vendor updates

vendor/k8s.io/kubernetes/cmd/BUILD

@@ -13,18 +13,19 @@ filegroup(
         ":package-srcs",
         "//cmd/clicheck:all-srcs",
         "//cmd/cloud-controller-manager:all-srcs",
+        "//cmd/controller-manager/app:all-srcs",
         "//cmd/gendocs:all-srcs",
         "//cmd/genkubedocs:all-srcs",
         "//cmd/genman:all-srcs",
         "//cmd/genswaggertypedocs:all-srcs",
         "//cmd/genutils:all-srcs",
         "//cmd/genyaml:all-srcs",
-        "//cmd/gke-certificates-controller:all-srcs",
         "//cmd/hyperkube:all-srcs",
         "//cmd/importverifier:all-srcs",
         "//cmd/kube-apiserver:all-srcs",
         "//cmd/kube-controller-manager:all-srcs",
         "//cmd/kube-proxy:all-srcs",
+        "//cmd/kube-scheduler:all-srcs",
         "//cmd/kubeadm:all-srcs",
         "//cmd/kubectl:all-srcs",
         "//cmd/kubelet:all-srcs",

vendor/k8s.io/kubernetes/cmd/clicheck/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "clicheck",
-    importpath = "k8s.io/kubernetes/cmd/clicheck",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/BUILD

@@ -9,14 +9,8 @@ load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "cloud-controller-manager",
-    gc_linkopts = [
-        "-linkmode",
-        "external",
-        "-extldflags",
-        "-static",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
+    pure = "on",
     x_defs = version_x_defs(),
 )
 
@@ -26,11 +20,9 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager",
     deps = [
         "//cmd/cloud-controller-manager/app:go_default_library",
-        "//cmd/cloud-controller-manager/app/options:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
         "//pkg/cloudprovider/providers:go_default_library",
         "//pkg/version/prometheus:go_default_library",
-        "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/BUILD

@@ -1,35 +1,29 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "go_default_library",
     srcs = ["controllermanager.go"],
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app",
+    visibility = ["//visibility:public"],
     deps = [
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
         "//cmd/cloud-controller-manager/app/options:go_default_library",
-        "//pkg/api/legacyscheme:go_default_library",
+        "//cmd/controller-manager/app:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller:go_default_library",
         "//pkg/controller/cloud:go_default_library",
         "//pkg/controller/route:go_default_library",
         "//pkg/controller/service:go_default_library",
         "//pkg/util/configz:go_default_library",
+        "//pkg/util/flag:go_default_library",
+        "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
@@ -47,7 +41,9 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//cmd/cloud-controller-manager/app/config:all-srcs",
         "//cmd/cloud-controller-manager/app/options:all-srcs",
     ],
     tags = ["automanaged"],
+    visibility = ["//visibility:public"],
 )

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/BUILD

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config",
+    visibility = ["//visibility:public"],
+    deps = ["//cmd/controller-manager/app:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/config/config.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"time"
+
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+)
+
+// ExtraConfig are part of Config, also can place your custom config here.
+type ExtraConfig struct {
+	NodeStatusUpdateFrequency time.Duration
+}
+
+// Config is the main context object for the cloud controller manager.
+type Config struct {
+	Generic genericcontrollermanager.Config
+	Extra   ExtraConfig
+}
+
+type completedConfig struct {
+	Generic genericcontrollermanager.CompletedConfig
+	Extra   *ExtraConfig
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{
+		c.Generic.Complete(),
+		&c.Extra,
+	}
+
+	return &CompletedConfig{&cc}
+}

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/controllermanager.go

@@ -17,41 +17,35 @@ limitations under the License.
 package app
 
 import (
+	"fmt"
 	"math/rand"
 	"net"
-	"net/http"
-	"net/http/pprof"
 	"os"
-	goruntime "runtime"
-	"strconv"
 	"strings"
 	"time"
 
-	"k8s.io/api/core/v1"
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
-	clientset "k8s.io/client-go/kubernetes"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/tools/record"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
 	cloudcontrollers "k8s.io/kubernetes/pkg/controller/cloud"
 	routecontroller "k8s.io/kubernetes/pkg/controller/route"
 	servicecontroller "k8s.io/kubernetes/pkg/controller/service"
 	"k8s.io/kubernetes/pkg/util/configz"
-
-	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	"k8s.io/kubernetes/pkg/version/verflag"
 )
 
 const (
@@ -61,94 +55,101 @@ const (
 
 // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
 func NewCloudControllerManagerCommand() *cobra.Command {
-	s := options.NewCloudControllerManagerServer()
-	s.AddFlags(pflag.CommandLine)
+	s := options.NewCloudControllerManagerOptions()
 	cmd := &cobra.Command{
 		Use: "cloud-controller-manager",
 		Long: `The Cloud controller manager is a daemon that embeds
 the cloud specific control loops shipped with Kubernetes.`,
 		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
+			c, err := s.Config()
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
+			if err := Run(c.Complete()); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
 		},
 	}
+	s.AddFlags(cmd.Flags())
 
 	return cmd
 }
 
 // resyncPeriod computes the time interval a shared informer waits before resyncing with the api server
-func resyncPeriod(s *options.CloudControllerManagerServer) func() time.Duration {
+func resyncPeriod(c *cloudcontrollerconfig.CompletedConfig) func() time.Duration {
 	return func() time.Duration {
 		factor := rand.Float64() + 1
-		return time.Duration(float64(s.MinResyncPeriod.Nanoseconds()) * factor)
+		return time.Duration(float64(c.Generic.ComponentConfig.MinResyncPeriod.Nanoseconds()) * factor)
 	}
 }
 
 // Run runs the ExternalCMServer.  This should never exit.
-func Run(s *options.CloudControllerManagerServer) error {
-	if s.CloudProvider == "" {
-		glog.Fatalf("--cloud-provider cannot be empty")
-	}
-
-	cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+func Run(c *cloudcontrollerconfig.CompletedConfig) error {
+	cloud, err := cloudprovider.InitCloudProvider(c.Generic.ComponentConfig.CloudProvider, c.Generic.ComponentConfig.CloudConfigFile)
 	if err != nil {
 		glog.Fatalf("Cloud provider could not be initialized: %v", err)
 	}
+	if cloud == nil {
+		glog.Fatalf("cloud provider is nil")
+	}
 
 	if cloud.HasClusterID() == false {
-		if s.AllowUntaggedCloud == true {
+		if c.Generic.ComponentConfig.AllowUntaggedCloud == true {
 			glog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
 		} else {
 			glog.Fatalf("no ClusterID found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
 		}
 	}
 
-	if c, err := configz.New("componentconfig"); err == nil {
-		c.Set(s.KubeControllerManagerConfiguration)
+	// setup /configz endpoint
+	if cz, err := configz.New("componentconfig"); err == nil {
+		cz.Set(c.Generic.ComponentConfig)
 	} else {
-		glog.Errorf("unable to register configz: %s", err)
-	}
-	kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
-	if err != nil {
-		return err
+		glog.Errorf("unable to register configz: %c", err)
 	}
 
-	// Set the ContentType of the requests from kube client
-	kubeconfig.ContentConfig.ContentType = s.ContentType
-	// Override kubeconfig qps/burst settings from flags
-	kubeconfig.QPS = s.KubeAPIQPS
-	kubeconfig.Burst = int(s.KubeAPIBurst)
-	kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "cloud-controller-manager"))
-	if err != nil {
-		glog.Fatalf("Invalid API configuration: %v", err)
+	// Start the controller manager HTTP server
+	stopCh := make(chan struct{})
+	if c.Generic.SecureServing != nil {
+		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh); err != nil {
+			return err
+		}
+	}
+	if c.Generic.InsecureServing != nil {
+		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
+			return err
+		}
 	}
-	leaderElectionClient := kubernetes.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
-
-	// Start the external controller manager server
-	go startHTTP(s)
-
-	recorder := createRecorder(kubeClient)
 
 	run := func(stop <-chan struct{}) {
 		rootClientBuilder := controller.SimpleControllerClientBuilder{
-			ClientConfig: kubeconfig,
+			ClientConfig: c.Generic.Kubeconfig,
 		}
 		var clientBuilder controller.ControllerClientBuilder
-		if s.UseServiceAccountCredentials {
+		if c.Generic.ComponentConfig.UseServiceAccountCredentials {
 			clientBuilder = controller.SAControllerClientBuilder{
-				ClientConfig:         restclient.AnonymousClientConfig(kubeconfig),
-				CoreClient:           kubeClient.CoreV1(),
-				AuthenticationClient: kubeClient.Authentication(),
+				ClientConfig:         restclient.AnonymousClientConfig(c.Generic.Kubeconfig),
+				CoreClient:           c.Generic.Client.CoreV1(),
+				AuthenticationClient: c.Generic.Client.AuthenticationV1(),
 				Namespace:            "kube-system",
 			}
 		} else {
 			clientBuilder = rootClientBuilder
 		}
 
-		err := StartControllers(s, kubeconfig, clientBuilder, stop, recorder, cloud)
-		glog.Fatalf("error running controllers: %v", err)
-		panic("unreachable")
+		if err := startControllers(c, c.Generic.Kubeconfig, rootClientBuilder, clientBuilder, stop, c.Generic.EventRecorder, cloud); err != nil {
+			glog.Fatalf("error running controllers: %v", err)
+		}
 	}
 
-	if !s.LeaderElection.LeaderElect {
+	if !c.Generic.ComponentConfig.LeaderElection.LeaderElect {
 		run(nil)
 		panic("unreachable")
 	}
@@ -158,15 +159,17 @@ func Run(s *options.CloudControllerManagerServer) error {
 	if err != nil {
 		return err
 	}
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id = id + "_" + string(uuid.NewUUID())
 
 	// Lock required for leader election
-	rl, err := resourcelock.New(s.LeaderElection.ResourceLock,
+	rl, err := resourcelock.New(c.Generic.ComponentConfig.LeaderElection.ResourceLock,
 		"kube-system",
 		"cloud-controller-manager",
-		leaderElectionClient.CoreV1(),
+		c.Generic.LeaderElectionClient.CoreV1(),
 		resourcelock.ResourceLockConfig{
-			Identity:      id + "-external-cloud-controller",
-			EventRecorder: recorder,
+			Identity:      id,
+			EventRecorder: c.Generic.EventRecorder,
 		})
 	if err != nil {
 		glog.Fatalf("error creating lock: %v", err)
@@ -175,9 +178,9 @@ func Run(s *options.CloudControllerManagerServer) error {
 	// Try and become the leader and start cloud controller manager loops
 	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
 		Lock:          rl,
-		LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
-		RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
-		RetryPeriod:   s.LeaderElection.RetryPeriod.Duration,
+		LeaseDuration: c.Generic.ComponentConfig.LeaderElection.LeaseDuration.Duration,
+		RenewDeadline: c.Generic.ComponentConfig.LeaderElection.RenewDeadline.Duration,
+		RetryPeriod:   c.Generic.ComponentConfig.LeaderElection.RetryPeriod.Duration,
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: run,
 			OnStoppedLeading: func() {
@@ -188,36 +191,36 @@ func Run(s *options.CloudControllerManagerServer) error {
 	panic("unreachable")
 }
 
-// StartControllers starts the cloud specific controller loops.
-func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restclient.Config, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
+// startControllers starts the cloud specific controller loops.
+func startControllers(c *cloudcontrollerconfig.CompletedConfig, kubeconfig *restclient.Config, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}, recorder record.EventRecorder, cloud cloudprovider.Interface) error {
 	// Function to build the kube client object
-	client := func(serviceAccountName string) clientset.Interface {
+	client := func(serviceAccountName string) kubernetes.Interface {
 		return clientBuilder.ClientOrDie(serviceAccountName)
 	}
-
 	if cloud != nil {
 		// Initialize the cloud provider with a reference to the clientBuilder
 		cloud.Initialize(clientBuilder)
 	}
 
-	versionedClient := client("shared-informers")
-	sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(s)())
+	// TODO: move this setup into Config
+	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
+	sharedInformers := informers.NewSharedInformerFactory(versionedClient, resyncPeriod(c)())
 
 	// Start the CloudNodeController
 	nodeController := cloudcontrollers.NewCloudNodeController(
 		sharedInformers.Core().V1().Nodes(),
 		client("cloud-node-controller"), cloud,
-		s.NodeMonitorPeriod.Duration,
-		s.NodeStatusUpdateFrequency.Duration)
+		c.Generic.ComponentConfig.NodeMonitorPeriod.Duration,
+		c.Extra.NodeStatusUpdateFrequency)
 
 	nodeController.Run()
-	time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
+	time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
 
 	// Start the PersistentVolumeLabelController
 	pvlController := cloudcontrollers.NewPersistentVolumeLabelController(client("pvl-controller"), cloud)
 	threads := 5
 	go pvlController.Run(threads, stop)
-	time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
+	time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
 
 	// Start the service controller
 	serviceController, err := servicecontroller.New(
@@ -225,79 +228,44 @@ func StartControllers(s *options.CloudControllerManagerServer, kubeconfig *restc
 		client("service-controller"),
 		sharedInformers.Core().V1().Services(),
 		sharedInformers.Core().V1().Nodes(),
-		s.ClusterName,
+		c.Generic.ComponentConfig.ClusterName,
 	)
 	if err != nil {
 		glog.Errorf("Failed to start service controller: %v", err)
 	} else {
-		go serviceController.Run(stop, int(s.ConcurrentServiceSyncs))
-		time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
+		go serviceController.Run(stop, int(c.Generic.ComponentConfig.ConcurrentServiceSyncs))
+		time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
 	}
 
 	// If CIDRs should be allocated for pods and set on the CloudProvider, then start the route controller
-	if s.AllocateNodeCIDRs && s.ConfigureCloudRoutes {
+	if c.Generic.ComponentConfig.AllocateNodeCIDRs && c.Generic.ComponentConfig.ConfigureCloudRoutes {
 		if routes, ok := cloud.Routes(); !ok {
 			glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
 		} else {
 			var clusterCIDR *net.IPNet
-			if len(strings.TrimSpace(s.ClusterCIDR)) != 0 {
-				_, clusterCIDR, err = net.ParseCIDR(s.ClusterCIDR)
+			if len(strings.TrimSpace(c.Generic.ComponentConfig.ClusterCIDR)) != 0 {
+				_, clusterCIDR, err = net.ParseCIDR(c.Generic.ComponentConfig.ClusterCIDR)
 				if err != nil {
-					glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", s.ClusterCIDR, err)
+					glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", c.Generic.ComponentConfig.ClusterCIDR, err)
 				}
 			}
 
-			routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), s.ClusterName, clusterCIDR)
-			go routeController.Run(stop, s.RouteReconciliationPeriod.Duration)
-			time.Sleep(wait.Jitter(s.ControllerStartInterval.Duration, ControllerStartJitter))
+			routeController := routecontroller.New(routes, client("route-controller"), sharedInformers.Core().V1().Nodes(), c.Generic.ComponentConfig.ClusterName, clusterCIDR)
+			go routeController.Run(stop, c.Generic.ComponentConfig.RouteReconciliationPeriod.Duration)
+			time.Sleep(wait.Jitter(c.Generic.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
 		}
 	} else {
-		glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", s.AllocateNodeCIDRs, s.ConfigureCloudRoutes)
+		glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", c.Generic.ComponentConfig.AllocateNodeCIDRs, c.Generic.ComponentConfig.ConfigureCloudRoutes)
 	}
 
 	// If apiserver is not running we should wait for some time and fail only then. This is particularly
 	// important when we start apiserver and controller manager at the same time.
-	err = wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
-		if _, err = restclient.ServerAPIVersions(kubeconfig); err == nil {
-			return true, nil
-		}
-		glog.Errorf("Failed to get api versions from server: %v", err)
-		return false, nil
-	})
+	err = genericcontrollermanager.WaitForAPIServer(versionedClient, 10*time.Second)
 	if err != nil {
-		glog.Fatalf("Failed to get api versions from server: %v", err)
+		glog.Fatalf("Failed to wait for apiserver being healthy: %v", err)
 	}
 
 	sharedInformers.Start(stop)
 
 	select {}
 }
-
-func startHTTP(s *options.CloudControllerManagerServer) {
-	mux := http.NewServeMux()
-	healthz.InstallHandler(mux)
-	if s.EnableProfiling {
-		mux.HandleFunc("/debug/pprof/", pprof.Index)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		if s.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-	configz.InstallHandler(mux)
-	mux.Handle("/metrics", prometheus.Handler())
-
-	server := &http.Server{
-		Addr:    net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))),
-		Handler: mux,
-	}
-	glog.Fatal(server.ListenAndServe())
-}
-
-func createRecorder(kubeClient *clientset.Clientset) record.EventRecorder {
-	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartLogging(glog.Infof)
-	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
-	return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "cloud-controller-manager"})
-}

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/BUILD

@@ -11,12 +11,14 @@ go_library(
     srcs = ["options.go"],
     importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
     deps = [
-        "//pkg/apis/componentconfig:go_default_library",
+        "//cmd/cloud-controller-manager/app/config:go_default_library",
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
     ],
 )
@@ -37,12 +39,13 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = ["options_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/cloud-controller-manager/app/options",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
     ],
 )

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/options.go

@@ -17,11 +17,14 @@ limitations under the License.
 package options
 
 import (
+	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	cloudcontrollerconfig "k8s.io/kubernetes/cmd/cloud-controller-manager/app/config"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/master/ports"
 
@@ -31,74 +34,81 @@ import (
 	"github.com/spf13/pflag"
 )
 
-// CloudControllerManagerServer is the main context object for the controller manager.
-type CloudControllerManagerServer struct {
-	componentconfig.KubeControllerManagerConfiguration
-
-	Master     string
-	Kubeconfig string
+// CloudControllerManagerOptions is the main context object for the controller manager.
+type CloudControllerManagerOptions struct {
+	Generic cmoptions.GenericControllerManagerOptions
 
 	// NodeStatusUpdateFrequency is the frequency at which the controller updates nodes' status
 	NodeStatusUpdateFrequency metav1.Duration
 }
 
-// NewCloudControllerManagerServer creates a new ExternalCMServer with a default config.
-func NewCloudControllerManagerServer() *CloudControllerManagerServer {
-	s := CloudControllerManagerServer{
-		// Part of these default values also present in 'cmd/kube-controller-manager/app/options/options.go'.
-		// Please keep them in sync when doing update.
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Port:                      ports.CloudControllerManagerPort,
-			Address:                   "0.0.0.0",
-			ConcurrentServiceSyncs:    1,
-			MinResyncPeriod:           metav1.Duration{Duration: 12 * time.Hour},
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:               "kubernetes",
-			ConfigureCloudRoutes:      true,
-			ContentType:               "application/vnd.kubernetes.protobuf",
-			KubeAPIQPS:                20.0,
-			KubeAPIBurst:              30,
-			LeaderElection:            leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			ControllerStartInterval:   metav1.Duration{Duration: 0 * time.Second},
-			RouteReconciliationPeriod: metav1.Duration{Duration: 10 * time.Second},
-		},
+// NewCloudControllerManagerOptions creates a new ExternalCMServer with a default config.
+func NewCloudControllerManagerOptions() *CloudControllerManagerOptions {
+	componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureCloudControllerManagerPort)
+
+	s := CloudControllerManagerOptions{
+		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
+		// Please make common changes there and put anything cloud specific here.
+		Generic:                   cmoptions.NewGenericControllerManagerOptions(componentConfig),
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 5 * time.Minute},
 	}
-	s.LeaderElection.LeaderElect = true
+	s.Generic.ComponentConfig.LeaderElection.LeaderElect = true
+
+	s.Generic.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes"
+	s.Generic.SecureServing.ServerCert.PairName = "cloud-controller-manager"
+
 	return &s
 }
 
 // AddFlags adds flags for a specific ExternalCMServer to the specified FlagSet
-func (s *CloudControllerManagerServer) AddFlags(fs *pflag.FlagSet) {
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the cloud-controller-manager's http service runs on.")
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces).")
-	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider of cloud services. Cannot be empty.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
-	fs.DurationVar(&s.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", s.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
+func (o *CloudControllerManagerOptions) AddFlags(fs *pflag.FlagSet) {
+	o.Generic.AddFlags(fs)
+
+	fs.StringVar(&o.Generic.ComponentConfig.CloudProvider, "cloud-provider", o.Generic.ComponentConfig.CloudProvider, "The provider of cloud services. Cannot be empty.")
+	fs.DurationVar(&o.NodeStatusUpdateFrequency.Duration, "node-status-update-frequency", o.NodeStatusUpdateFrequency.Duration, "Specifies how often the controller updates nodes' status.")
 	// TODO: remove --service-account-private-key-file 6 months after 1.8 is released (~1.10)
-	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
+	fs.StringVar(&o.Generic.ComponentConfig.ServiceAccountKeyFile, "service-account-private-key-file", o.Generic.ComponentConfig.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
 	fs.MarkDeprecated("service-account-private-key-file", "This flag is currently no-op and will be deleted.")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/.")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster.")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster.")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
-	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
-	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
+	fs.Int32Var(&o.Generic.ComponentConfig.ConcurrentServiceSyncs, "concurrent-service-syncs", o.Generic.ComponentConfig.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
+
+	leaderelectionconfig.BindFlags(&o.Generic.ComponentConfig.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }
+
+// ApplyTo fills up cloud controller manager config with options.
+func (o *CloudControllerManagerOptions) ApplyTo(c *cloudcontrollerconfig.Config) error {
+	if err := o.Generic.ApplyTo(&c.Generic, "cloud-controller-manager"); err != nil {
+		return err
+	}
+
+	c.Extra.NodeStatusUpdateFrequency = o.NodeStatusUpdateFrequency.Duration
+
+	return nil
+}
+
+// Validate is used to validate config before launching the cloud controller manager
+func (o *CloudControllerManagerOptions) Validate() error {
+	errors := []error{}
+	errors = append(errors, o.Generic.Validate()...)
+
+	if len(o.Generic.ComponentConfig.CloudProvider) == 0 {
+		errors = append(errors, fmt.Errorf("--cloud-provider cannot be empty"))
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Config return a cloud controller manager config objective
+func (o CloudControllerManagerOptions) Config() (*cloudcontrollerconfig.Config, error) {
+	if err := o.Validate(); err != nil {
+		return nil, err
+	}
+
+	c := &cloudcontrollerconfig.Config{}
+	if err := o.ApplyTo(c); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/app/options/options_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"net"
 	"reflect"
 	"testing"
 	"time"
@@ -25,12 +26,14 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 )
 
 func TestAddFlags(t *testing.T) {
 	f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
-	s := NewCloudControllerManagerServer()
+	s := NewCloudControllerManagerOptions()
 	s.AddFlags(f)
 
 	args := []string{
@@ -43,6 +46,7 @@ func TestAddFlags(t *testing.T) {
 		"--configure-cloud-routes=false",
 		"--contention-profiling=true",
 		"--controller-start-interval=2m",
+		"--http2-max-streams-per-connection=47",
 		"--min-resync-period=5m",
 		"--kube-api-burst=100",
 		"--kube-api-content-type=application/vnd.kubernetes.protobuf",
@@ -61,38 +65,102 @@ func TestAddFlags(t *testing.T) {
 		"--route-reconciliation-period=30s",
 		"--min-resync-period=100m",
 		"--use-service-account-credentials=false",
+		"--cert-dir=/a/b/c",
+		"--bind-address=192.168.4.21",
+		"--secure-port=10001",
 	}
 	f.Parse(args)
 
-	expected := &CloudControllerManagerServer{
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			CloudProvider:             "gce",
-			CloudConfigFile:           "/cloud-config",
-			Port:                      10000,
-			Address:                   "192.168.4.10",
-			ConcurrentServiceSyncs:    1,
-			MinResyncPeriod:           metav1.Duration{Duration: 100 * time.Minute},
-			NodeMonitorPeriod:         metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:               "k8s",
-			ConfigureCloudRoutes:      false,
-			AllocateNodeCIDRs:         true,
-			ContentType:               "application/vnd.kubernetes.protobuf",
-			EnableContentionProfiling: true,
-			KubeAPIQPS:                50.0,
-			KubeAPIBurst:              100,
-			LeaderElection: componentconfig.LeaderElectionConfiguration{
-				ResourceLock:  "configmap",
-				LeaderElect:   false,
-				LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
-				RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
-				RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+	expected := &CloudControllerManagerOptions{
+		Generic: cmoptions.GenericControllerManagerOptions{
+			ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
+				CloudProvider:                                   "gce",
+				CloudConfigFile:                                 "/cloud-config",
+				Port:                                            10253,     // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				Address:                                         "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				ConcurrentEndpointSyncs:                         5,
+				ConcurrentRSSyncs:                               5,
+				ConcurrentResourceQuotaSyncs:                    5,
+				ConcurrentDeploymentSyncs:                       5,
+				ConcurrentDaemonSetSyncs:                        2,
+				ConcurrentJobSyncs:                              5,
+				ConcurrentNamespaceSyncs:                        10,
+				ConcurrentSATokenSyncs:                          5,
+				ConcurrentServiceSyncs:                          1,
+				ConcurrentGCSyncs:                               20,
+				ConcurrentRCSyncs:                               5,
+				MinResyncPeriod:                                 metav1.Duration{Duration: 100 * time.Minute},
+				NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
+				ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+				NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+				PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+				HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+				HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+				HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+				HorizontalPodAutoscalerTolerance:                0.1,
+				DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+				PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+				NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+				NodeStartupGracePeriod:                          metav1.Duration{Duration: 1 * time.Minute},
+				ClusterSigningDuration:                          metav1.Duration{Duration: 8760 * time.Hour},
+				ReconcilerSyncLoopPeriod:                        metav1.Duration{Duration: 1 * time.Minute},
+				TerminatedPodGCThreshold:                        12500,
+				RegisterRetryCount:                              10,
+				ClusterName:                                     "k8s",
+				ConfigureCloudRoutes:                            false,
+				AllocateNodeCIDRs:                               true,
+				EnableGarbageCollector:                          true,
+				EnableTaintManager:                              true,
+				HorizontalPodAutoscalerUseRESTClients:           true,
+				VolumeConfiguration: componentconfig.VolumeConfiguration{
+					EnableDynamicProvisioning:  true,
+					EnableHostPathProvisioning: false,
+					FlexVolumePluginDir:        "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+					PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+						MaximumRetry:             3,
+						MinimumTimeoutNFS:        300,
+						IncrementTimeoutNFS:      30,
+						MinimumTimeoutHostPath:   60,
+						IncrementTimeoutHostPath: 30,
+					},
+				},
+				ContentType:               "application/vnd.kubernetes.protobuf",
+				ClusterSigningCertFile:    "/etc/kubernetes/ca/ca.pem",
+				ClusterSigningKeyFile:     "/etc/kubernetes/ca/ca.key",
+				EnableContentionProfiling: true,
+				KubeAPIQPS:                50.0,
+				KubeAPIBurst:              100,
+				LeaderElection: componentconfig.LeaderElectionConfiguration{
+					ResourceLock:  "configmap",
+					LeaderElect:   false,
+					LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+					RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+					RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				},
+				ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
+				RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
+				ClusterCIDR:               "1.2.3.4/24",
+				NodeCIDRMaskSize:          24,
+				CIDRAllocatorType:         "RangeAllocator",
+				Controllers:               []string{"*"},
 			},
-			ControllerStartInterval:   metav1.Duration{Duration: 2 * time.Minute},
-			RouteReconciliationPeriod: metav1.Duration{Duration: 30 * time.Second},
-			ClusterCIDR:               "1.2.3.4/24",
+			SecureServing: &apiserveroptions.SecureServingOptions{
+				BindPort:    10001,
+				BindAddress: net.ParseIP("192.168.4.21"),
+				ServerCert: apiserveroptions.GeneratableKeyCert{
+					CertDirectory: "/a/b/c",
+					PairName:      "cloud-controller-manager",
+				},
+				HTTP2MaxStreamsPerConnection: 47,
+			},
+			InsecureServing: &cmoptions.InsecureServingOptions{
+				BindAddress: net.ParseIP("192.168.4.10"),
+				BindPort:    int(10000),
+				BindNetwork: "tcp",
+			},
+			Kubeconfig: "/kubeconfig",
+			Master:     "192.168.4.20",
 		},
-		Kubeconfig: "/kubeconfig",
-		Master:     "192.168.4.20",
 		NodeStatusUpdateFrequency: metav1.Duration{Duration: 10 * time.Minute},
 	}
 	if !reflect.DeepEqual(expected, s) {

vendor/k8s.io/kubernetes/cmd/cloud-controller-manager/controller-manager.go

@@ -20,35 +20,40 @@ limitations under the License.
 package main
 
 import (
+	goflag "flag"
 	"fmt"
+	"math/rand"
 	"os"
+	"time"
 
-	"k8s.io/apiserver/pkg/util/flag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
-	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
 	// NOTE: Importing all in-tree cloud-providers is not required when
 	// implementing an out-of-tree cloud-provider.
 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
 	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
-	"k8s.io/kubernetes/pkg/version/verflag"
 
 	"github.com/spf13/pflag"
 )
 
 func main() {
-	s := options.NewCloudControllerManagerServer()
-	s.AddFlags(pflag.CommandLine)
+	rand.Seed(time.Now().UTC().UnixNano())
 
-	flag.InitFlags()
+	command := app.NewCloudControllerManagerCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	verflag.PrintAndExitIfRequested()
-
-	if err := app.Run(s); err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+	if err := command.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)
 	}
 }

vendor/k8s.io/kubernetes/cmd/controller-manager/OWNERS

@@ -0,0 +1,11 @@
+approvers:
+- deads2k
+- cheftako
+- sttts
+reviewers:
+- caesarxuchao
+- cheftako
+- deads2k
+- lavalamp
+- liggitt
+- sttts

vendor/k8s.io/kubernetes/cmd/controller-manager/app/BUILD

@@ -0,0 +1,48 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "helper.go",
+        "insecure_serving.go",
+        "serve.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/controller-manager/app",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/util/configz:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/filters:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/endpoints/request:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/mux:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/routes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//cmd/controller-manager/app/options:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/cmd/controller-manager/app/config.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	apiserver "k8s.io/apiserver/pkg/server"
+	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// Config is the main context object for the controller manager.
+type Config struct {
+	// TODO: split up the component config. This is not generic.
+	ComponentConfig componentconfig.KubeControllerManagerConfiguration
+
+	SecureServing *apiserver.SecureServingInfo
+	// TODO: remove deprecated insecure serving
+	InsecureServing *InsecureServingInfo
+	Authentication  apiserver.AuthenticationInfo
+	Authorization   apiserver.AuthorizationInfo
+
+	// the general kube client
+	Client *clientset.Clientset
+
+	// the client only used for leader election
+	LeaderElectionClient *clientset.Clientset
+
+	// the rest config for the master
+	Kubeconfig *restclient.Config
+
+	// the event sink
+	EventRecorder record.EventRecorder
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() CompletedConfig {
+	cc := completedConfig{c}
+	return CompletedConfig{&cc}
+}

vendor/k8s.io/kubernetes/cmd/controller-manager/app/helper.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+)
+
+// WaitForAPIServer waits for the API Server's /healthz endpoint to report "ok" with timeout.
+func WaitForAPIServer(client clientset.Interface, timeout time.Duration) error {
+	var lastErr error
+
+	err := wait.PollImmediate(time.Second, timeout, func() (bool, error) {
+		healthStatus := 0
+		result := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
+		if result.Error() != nil {
+			lastErr = fmt.Errorf("failed to get apiserver /healthz status: %v", result.Error())
+			return false, nil
+		}
+		if healthStatus != http.StatusOK {
+			content, _ := result.Raw()
+			lastErr = fmt.Errorf("APIServer isn't healthy: %v", string(content))
+			glog.Warningf("APIServer isn't healthy yet: %v. Waiting a little while.", string(content))
+			return false, nil
+		}
+
+		return true, nil
+	})
+
+	if err != nil {
+		return fmt.Errorf("%v: %v", err, lastErr)
+	}
+
+	return nil
+}

vendor/k8s.io/kubernetes/cmd/controller-manager/app/insecure_serving.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/golang/glog"
+
+	"k8s.io/apiserver/pkg/server"
+)
+
+// InsecureServingInfo is the main context object for the insecure http server.
+type InsecureServingInfo struct {
+	// Listener is the secure server network listener.
+	Listener net.Listener
+}
+
+// Serve starts an insecure http server with the given handler. It fails only if
+// the initial listen call fails. It does not block.
+func (s *InsecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error {
+	insecureServer := &http.Server{
+		Addr:           s.Listener.Addr().String(),
+		Handler:        handler,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	glog.Infof("Serving insecurely on %s", s.Listener.Addr())
+	return server.RunServer(insecureServer, s.Listener, shutdownTimeout, stopCh)
+}

vendor/k8s.io/kubernetes/cmd/controller-manager/app/options/BUILD

@@ -0,0 +1,42 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "insecure_serving.go",
+        "options.go",
+    ],
+    importpath = "k8s.io/kubernetes/cmd/controller-manager/app/options",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//cmd/controller-manager/app:go_default_library",
+        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/client/leaderelectionconfig:go_default_library",
+        "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/cmd/controller-manager/app/options/insecure_serving.go

@@ -0,0 +1,112 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/server/options"
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+)
+
+// InsecureServingOptions are for creating an unauthenticated, unauthorized, insecure port.
+// No one should be using these anymore.
+type InsecureServingOptions struct {
+	BindAddress net.IP
+	BindPort    int
+	// BindNetwork is the type of network to bind to - defaults to "tcp", accepts "tcp",
+	// "tcp4", and "tcp6".
+	BindNetwork string
+
+	// Listener is the secure server network listener.
+	// either Listener or BindAddress/BindPort/BindNetwork is set,
+	// if Listener is set, use it and omit BindAddress/BindPort/BindNetwork.
+	Listener net.Listener
+}
+
+// Validate ensures that the insecure port values within the range of the port.
+func (s *InsecureServingOptions) Validate() []error {
+	errors := []error{}
+
+	if s == nil {
+		return nil
+	}
+
+	if s.BindPort < 0 || s.BindPort > 32767 {
+		errors = append(errors, fmt.Errorf("--insecure-port %v must be between 0 and 32767, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
+	}
+
+	return errors
+}
+
+// AddFlags adds flags related to insecure serving for controller manager to the specified FlagSet.
+func (s *InsecureServingOptions) AddFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+}
+
+// AddDeprecatedFlags adds deprecated flags related to insecure serving for controller manager to the specified FlagSet.
+// TODO: remove it until kops stop using `--address`
+func (s *InsecureServingOptions) AddDeprecatedFlags(fs *pflag.FlagSet) {
+	if s == nil {
+		return
+	}
+
+	fs.IPVar(&s.BindAddress, "address", s.BindAddress,
+		"DEPRECATED: the IP address on which to listen for the --port port. See --bind-address instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("address", "see --bind-address instead.")
+
+	fs.IntVar(&s.BindPort, "port", s.BindPort, "DEPRECATED: the port on which to serve HTTP insecurely without authentication and authorization. If 0, don't serve HTTPS at all. See --secure-port instead.")
+	// MarkDeprecated hides the flag from the help. We don't want that:
+	// fs.MarkDeprecated("port", "see --secure-port instead.")
+}
+
+// ApplyTo adds InsecureServingOptions to the insecureserverinfo amd kube-controller manager configuration.
+// Note: the double pointer allows to set the *InsecureServingInfo to nil without referencing the struct hosting this pointer.
+func (s *InsecureServingOptions) ApplyTo(c **genericcontrollermanager.InsecureServingInfo, cfg *componentconfig.KubeControllerManagerConfiguration) error {
+	if s == nil {
+		return nil
+	}
+	if s.BindPort <= 0 {
+		return nil
+	}
+
+	if s.Listener == nil {
+		var err error
+		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
+		s.Listener, s.BindPort, err = options.CreateListener(s.BindNetwork, addr)
+		if err != nil {
+			return fmt.Errorf("failed to create listener: %v", err)
+		}
+	}
+
+	*c = &genericcontrollermanager.InsecureServingInfo{
+		Listener: s.Listener,
+	}
+
+	// sync back to component config
+	// TODO: find more elegant way than synching back the values.
+	cfg.Port = int32(s.BindPort)
+	cfg.Address = s.BindAddress.String()
+
+	return nil
+}

vendor/k8s.io/kubernetes/cmd/controller-manager/app/options/options.go

@@ -0,0 +1,242 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"net"
+	"time"
+
+	"github.com/cloudflare/cfssl/helpers"
+	"github.com/golang/glog"
+
+	"github.com/spf13/pflag"
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	"k8s.io/client-go/kubernetes"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/record"
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
+)
+
+// GenericControllerManagerOptions is the common structure for a controller manager. It works with NewGenericControllerManagerOptions
+// and AddDefaultControllerFlags to create the common components of kube-controller-manager and cloud-controller-manager.
+type GenericControllerManagerOptions struct {
+	// TODO: turn ComponentConfig into modular option structs. This is not generic.
+	ComponentConfig componentconfig.KubeControllerManagerConfiguration
+
+	SecureServing *apiserveroptions.SecureServingOptions
+	// TODO: remove insecure serving mode
+	InsecureServing *InsecureServingOptions
+	Authentication  *apiserveroptions.DelegatingAuthenticationOptions
+	Authorization   *apiserveroptions.DelegatingAuthorizationOptions
+
+	Master     string
+	Kubeconfig string
+}
+
+const (
+	// These defaults are deprecated and exported so that we can warn if
+	// they are being used.
+
+	// DefaultClusterSigningCertFile is deprecated. Do not use.
+	DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
+	// DefaultClusterSigningKeyFile is deprecated. Do not use.
+	DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
+)
+
+// NewGenericControllerManagerOptions returns common/default configuration values for both
+// the kube-controller-manager and the cloud-contoller-manager. Any common changes should
+// be made here. Any individual changes should be made in that controller.
+func NewGenericControllerManagerOptions(componentConfig componentconfig.KubeControllerManagerConfiguration) GenericControllerManagerOptions {
+	o := GenericControllerManagerOptions{
+		ComponentConfig: componentConfig,
+		SecureServing:   apiserveroptions.NewSecureServingOptions(),
+		InsecureServing: &InsecureServingOptions{
+			BindAddress: net.ParseIP(componentConfig.Address),
+			BindPort:    int(componentConfig.Port),
+			BindNetwork: "tcp",
+		},
+		Authentication: nil, // TODO: enable with apiserveroptions.NewDelegatingAuthenticationOptions()
+		Authorization:  nil, // TODO: enable with apiserveroptions.NewDelegatingAuthorizationOptions()
+	}
+
+	// disable secure serving for now
+	// TODO: enable HTTPS by default
+	o.SecureServing.BindPort = 0
+
+	return o
+}
+
+// NewDefaultControllerManagerComponentConfig returns default kube-controller manager configuration object.
+func NewDefaultControllerManagerComponentConfig(insecurePort int32) componentconfig.KubeControllerManagerConfiguration {
+	return componentconfig.KubeControllerManagerConfiguration{
+		Controllers:                                     []string{"*"},
+		Port:                                            insecurePort,
+		Address:                                         "0.0.0.0",
+		ConcurrentEndpointSyncs:                         5,
+		ConcurrentServiceSyncs:                          1,
+		ConcurrentRCSyncs:                               5,
+		ConcurrentRSSyncs:                               5,
+		ConcurrentDaemonSetSyncs:                        2,
+		ConcurrentJobSyncs:                              5,
+		ConcurrentResourceQuotaSyncs:                    5,
+		ConcurrentDeploymentSyncs:                       5,
+		ConcurrentNamespaceSyncs:                        10,
+		ConcurrentSATokenSyncs:                          5,
+		RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
+		ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
+		NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
+		PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
+		HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
+		HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
+		HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
+		HorizontalPodAutoscalerTolerance:                0.1,
+		DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
+		MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
+		RegisterRetryCount:                              10,
+		PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
+		NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
+		NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
+		NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
+		ClusterName:                                     "kubernetes",
+		NodeCIDRMaskSize:                                24,
+		ConfigureCloudRoutes:                            true,
+		TerminatedPodGCThreshold:                        12500,
+		VolumeConfiguration: componentconfig.VolumeConfiguration{
+			EnableHostPathProvisioning: false,
+			EnableDynamicProvisioning:  true,
+			PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+				MaximumRetry:             3,
+				MinimumTimeoutNFS:        300,
+				IncrementTimeoutNFS:      30,
+				MinimumTimeoutHostPath:   60,
+				IncrementTimeoutHostPath: 30,
+			},
+			FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
+		},
+		ContentType:                           "application/vnd.kubernetes.protobuf",
+		KubeAPIQPS:                            20.0,
+		KubeAPIBurst:                          30,
+		LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
+		ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
+		EnableGarbageCollector:                true,
+		ConcurrentGCSyncs:                     20,
+		ClusterSigningCertFile:                DefaultClusterSigningCertFile,
+		ClusterSigningKeyFile:                 DefaultClusterSigningKeyFile,
+		ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
+		ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
+		EnableTaintManager:                    true,
+		HorizontalPodAutoscalerUseRESTClients: true,
+	}
+}
+
+// AddFlags adds common/default flags for both the kube and cloud Controller Manager Server to the
+// specified FlagSet. Any common changes should be made here. Any individual changes should be made in that controller.
+func (o *GenericControllerManagerOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&o.ComponentConfig.UseServiceAccountCredentials, "use-service-account-credentials", o.ComponentConfig.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
+	fs.StringVar(&o.ComponentConfig.CloudConfigFile, "cloud-config", o.ComponentConfig.CloudConfigFile, "The path to the cloud provider configuration file. Empty string for no configuration file.")
+	fs.BoolVar(&o.ComponentConfig.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances. This is a legacy mode of operation and a cluster-id will be required in the future.")
+	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release. A cluster-id will be required on cloud instances.")
+	fs.DurationVar(&o.ComponentConfig.RouteReconciliationPeriod.Duration, "route-reconciliation-period", o.ComponentConfig.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
+	fs.DurationVar(&o.ComponentConfig.MinResyncPeriod.Duration, "min-resync-period", o.ComponentConfig.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod.")
+	fs.DurationVar(&o.ComponentConfig.NodeMonitorPeriod.Duration, "node-monitor-period", o.ComponentConfig.NodeMonitorPeriod.Duration,
+		"The period for syncing NodeStatus in NodeController.")
+	fs.BoolVar(&o.ComponentConfig.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&o.ComponentConfig.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled.")
+	fs.StringVar(&o.ComponentConfig.ClusterName, "cluster-name", o.ComponentConfig.ClusterName, "The instance prefix for the cluster.")
+	fs.StringVar(&o.ComponentConfig.ClusterCIDR, "cluster-cidr", o.ComponentConfig.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
+	fs.BoolVar(&o.ComponentConfig.AllocateNodeCIDRs, "allocate-node-cidrs", false, "Should CIDRs for Pods be allocated and set on the cloud provider.")
+	fs.StringVar(&o.ComponentConfig.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator", "Type of CIDR allocator to use")
+	fs.BoolVar(&o.ComponentConfig.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
+	fs.StringVar(&o.Master, "master", o.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig).")
+	fs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
+	fs.StringVar(&o.ComponentConfig.ContentType, "kube-api-content-type", o.ComponentConfig.ContentType, "Content type of requests sent to apiserver.")
+	fs.Float32Var(&o.ComponentConfig.KubeAPIQPS, "kube-api-qps", o.ComponentConfig.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver.")
+	fs.Int32Var(&o.ComponentConfig.KubeAPIBurst, "kube-api-burst", o.ComponentConfig.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver.")
+	fs.DurationVar(&o.ComponentConfig.ControllerStartInterval.Duration, "controller-start-interval", o.ComponentConfig.ControllerStartInterval.Duration, "Interval between starting controller managers.")
+
+	o.SecureServing.AddFlags(fs)
+	o.InsecureServing.AddFlags(fs)
+	o.InsecureServing.AddDeprecatedFlags(fs)
+	o.Authentication.AddFlags(fs)
+	o.Authorization.AddFlags(fs)
+}
+
+// ApplyTo fills up controller manager config with options and userAgent
+func (o *GenericControllerManagerOptions) ApplyTo(c *genericcontrollermanager.Config, userAgent string) error {
+	c.ComponentConfig = o.ComponentConfig
+
+	if err := o.SecureServing.ApplyTo(&c.SecureServing); err != nil {
+		return err
+	}
+	if err := o.InsecureServing.ApplyTo(&c.InsecureServing, &c.ComponentConfig); err != nil {
+		return err
+	}
+	if err := o.Authentication.ApplyTo(&c.Authentication, c.SecureServing, nil); err != nil {
+		return err
+	}
+	if err := o.Authorization.ApplyTo(&c.Authorization); err != nil {
+		return err
+	}
+
+	var err error
+	c.Kubeconfig, err = clientcmd.BuildConfigFromFlags(o.Master, o.Kubeconfig)
+	if err != nil {
+		return err
+	}
+	c.Kubeconfig.ContentConfig.ContentType = o.ComponentConfig.ContentType
+	c.Kubeconfig.QPS = o.ComponentConfig.KubeAPIQPS
+	c.Kubeconfig.Burst = int(o.ComponentConfig.KubeAPIBurst)
+
+	c.Client, err = clientset.NewForConfig(restclient.AddUserAgent(c.Kubeconfig, userAgent))
+	if err != nil {
+		return err
+	}
+
+	c.LeaderElectionClient = clientset.NewForConfigOrDie(restclient.AddUserAgent(c.Kubeconfig, "leader-election"))
+
+	c.EventRecorder = createRecorder(c.Client, userAgent)
+
+	return nil
+}
+
+// Validate checks GenericControllerManagerOptions and return a slice of found errors.
+func (o *GenericControllerManagerOptions) Validate() []error {
+	errors := []error{}
+	errors = append(errors, o.SecureServing.Validate()...)
+	errors = append(errors, o.InsecureServing.Validate()...)
+	errors = append(errors, o.Authentication.Validate()...)
+	errors = append(errors, o.Authorization.Validate()...)
+
+	// TODO: validate component config, master and kubeconfig
+
+	return errors
+}
+
+func createRecorder(kubeClient *kubernetes.Clientset, userAgent string) record.EventRecorder {
+	eventBroadcaster := record.NewBroadcaster()
+	eventBroadcaster.StartLogging(glog.Infof)
+	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
+	return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: userAgent})
+}

vendor/k8s.io/kubernetes/cmd/controller-manager/app/serve.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"net/http"
+	goruntime "runtime"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	genericfilters "k8s.io/apiserver/pkg/server/filters"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/mux"
+	"k8s.io/apiserver/pkg/server/routes"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/util/configz"
+)
+
+type serveFunc func(handler http.Handler, shutdownTimeout time.Duration, stopCh <-chan struct{}) error
+
+// Serve creates a base handler chain for a controller manager. It runs the
+// the chain with the given serveFunc.
+func Serve(c *CompletedConfig, serveFunc serveFunc, stopCh <-chan struct{}) error {
+	mux := mux.NewPathRecorderMux("controller-manager")
+	healthz.InstallHandler(mux)
+	if c.ComponentConfig.EnableProfiling {
+		routes.Profiling{}.Install(mux)
+		if c.ComponentConfig.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+	configz.InstallHandler(mux)
+	mux.Handle("/metrics", prometheus.Handler())
+
+	requestContextMapper := apirequest.NewRequestContextMapper()
+	requestInfoResolver := &apirequest.RequestInfoFactory{}
+	failedHandler := genericapifilters.Unauthorized(requestContextMapper, legacyscheme.Codecs, false)
+
+	handler := genericapifilters.WithAuthorization(mux, requestContextMapper, c.Authorization.Authorizer, legacyscheme.Codecs)
+	handler = genericapifilters.WithAuthentication(handler, requestContextMapper, c.Authentication.Authenticator, failedHandler)
+	handler = genericapifilters.WithRequestInfo(handler, requestInfoResolver, requestContextMapper)
+	handler = apirequest.WithRequestContext(handler, requestContextMapper)
+	handler = genericfilters.WithPanicRecovery(handler)
+
+	return serveFunc(handler, 0, stopCh)
+}

vendor/k8s.io/kubernetes/cmd/gendocs/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "gendocs",
-    importpath = "k8s.io/kubernetes/cmd/gendocs",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(

vendor/k8s.io/kubernetes/cmd/genkubedocs/BUILD

@@ -9,8 +9,7 @@ load(
 
 go_binary(
     name = "genkubedocs",
-    importpath = "k8s.io/kubernetes/cmd/genkubedocs",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(
@@ -26,9 +25,9 @@ go_library(
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-controller-manager/app:go_default_library",
         "//cmd/kube-proxy/app:go_default_library",
+        "//cmd/kube-scheduler/app:go_default_library",
         "//cmd/kubeadm/app/cmd:go_default_library",
         "//cmd/kubelet/app:go_default_library",
-        "//plugin/cmd/kube-scheduler/app:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/cobra/doc:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
@@ -51,6 +50,5 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = ["postprocessing_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/genkubedocs",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )

vendor/k8s.io/kubernetes/cmd/genkubedocs/gen_kube_docs.go

@@ -27,9 +27,9 @@ import (
 	apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
 	proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app"
+	schapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
 	kubeadmapp "k8s.io/kubernetes/cmd/kubeadm/app/cmd"
 	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
-	schapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
 )
 
 func main() {

vendor/k8s.io/kubernetes/cmd/genkubedocs/postprocessing.go

@@ -27,7 +27,7 @@ import (
 // MarkdownPostProcessing goes though the generated files
 func MarkdownPostProcessing(cmd *cobra.Command, dir string, processor func(string) string) error {
 	for _, c := range cmd.Commands() {
-		if !c.IsAvailableCommand() || c.IsHelpCommand() {
+		if !c.IsAvailableCommand() || c.IsAdditionalHelpTopicCommand() {
 			continue
 		}
 		if err := MarkdownPostProcessing(c, dir, processor); err != nil {

vendor/k8s.io/kubernetes/cmd/genman/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "genman",
-    importpath = "k8s.io/kubernetes/cmd/genman",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(
@@ -22,11 +21,11 @@ go_library(
         "//cmd/kube-apiserver/app:go_default_library",
         "//cmd/kube-controller-manager/app:go_default_library",
         "//cmd/kube-proxy/app:go_default_library",
+        "//cmd/kube-scheduler/app:go_default_library",
         "//cmd/kubeadm/app/cmd:go_default_library",
         "//cmd/kubelet/app:go_default_library",
         "//pkg/kubectl/cmd:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
-        "//plugin/cmd/kube-scheduler/app:go_default_library",
         "//vendor/github.com/cpuguy83/go-md2man/md2man:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",

vendor/k8s.io/kubernetes/cmd/genman/gen_kube_man.go

@@ -31,11 +31,11 @@ import (
 	apiservapp "k8s.io/kubernetes/cmd/kube-apiserver/app"
 	cmapp "k8s.io/kubernetes/cmd/kube-controller-manager/app"
 	proxyapp "k8s.io/kubernetes/cmd/kube-proxy/app"
+	schapp "k8s.io/kubernetes/cmd/kube-scheduler/app"
 	kubeadmapp "k8s.io/kubernetes/cmd/kubeadm/app/cmd"
 	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
 	kubectlcmd "k8s.io/kubernetes/pkg/kubectl/cmd"
 	kubectlcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
-	schapp "k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
 )
 
 func main() {

vendor/k8s.io/kubernetes/cmd/genswaggertypedocs/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "genswaggertypedocs",
-    importpath = "k8s.io/kubernetes/cmd/genswaggertypedocs",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(

vendor/k8s.io/kubernetes/cmd/genutils/BUILD

@@ -15,8 +15,7 @@ go_library(
 go_test(
     name = "go_default_test",
     srcs = ["genutils_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/genutils",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 filegroup(

vendor/k8s.io/kubernetes/cmd/genyaml/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "genyaml",
-    importpath = "k8s.io/kubernetes/cmd/genyaml",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/OWNERS

@@ -1,8 +0,0 @@
-approvers:
-- pipejakob
-- mikedanese
-- roberthbailey
-reviewers:
-- pipejakob
-- mikedanese
-- roberthbailey

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/app/BUILD

@@ -1,62 +0,0 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
-
-go_library(
-    name = "go_default_library",
-    srcs = [
-        "gke_certificates_controller.go",
-        "gke_signer.go",
-        "options.go",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/gke-certificates-controller/app",
-    deps = [
-        "//pkg/api/legacyscheme:go_default_library",
-        "//pkg/apis/certificates/install:go_default_library",
-        "//pkg/controller:go_default_library",
-        "//pkg/controller/certificates:go_default_library",
-        "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/api/core/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/util/webhook:go_default_library",
-        "//vendor/k8s.io/client-go/informers:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
-        "//vendor/k8s.io/client-go/plugin/pkg/client/auth:go_default_library",
-        "//vendor/k8s.io/client-go/rest:go_default_library",
-        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-    ],
-)
-
-filegroup(
-    name = "package-srcs",
-    srcs = glob(["**"]),
-    tags = ["automanaged"],
-    visibility = ["//visibility:private"],
-)
-
-filegroup(
-    name = "all-srcs",
-    srcs = [":package-srcs"],
-    tags = ["automanaged"],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["gke_signer_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/gke-certificates-controller/app",
-    library = ":go_default_library",
-    deps = [
-        "//vendor/k8s.io/api/certificates/v1beta1:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
-    ],
-)

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/app/gke_certificates_controller.go

@@ -1,89 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package app implements a server that runs a stand-alone version of the
-// certificates controller for GKE clusters.
-package app
-
-import (
-	"time"
-
-	"k8s.io/api/core/v1"
-	"k8s.io/client-go/informers"
-	clientset "k8s.io/client-go/kubernetes"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
-	"k8s.io/kubernetes/pkg/controller"
-	"k8s.io/kubernetes/pkg/controller/certificates"
-
-	// Install all auth plugins
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-
-	"github.com/golang/glog"
-	"github.com/spf13/cobra"
-)
-
-// NewGKECertificatesControllerCommand creates a new *cobra.Command with default parameters.
-func NewGKECertificatesControllerCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use: "gke-certificates-controller",
-		Long: `The Kubernetes GKE certificates controller is a daemon that
-handles auto-approving and signing certificates for GKE clusters.`,
-	}
-
-	return cmd
-}
-
-// Run runs the GKECertificatesController. This should never exit.
-func Run(s *GKECertificatesController) error {
-	kubeconfig, err := clientcmd.BuildConfigFromFlags("", s.Kubeconfig)
-	if err != nil {
-		return err
-	}
-
-	kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "gke-certificates-controller"))
-	if err != nil {
-		return err
-	}
-
-	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartLogging(glog.Infof)
-	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
-	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "gke-certificates-controller"})
-
-	clientBuilder := controller.SimpleControllerClientBuilder{ClientConfig: kubeconfig}
-	client := clientBuilder.ClientOrDie("certificate-controller")
-
-	sharedInformers := informers.NewSharedInformerFactory(client, time.Duration(12)*time.Hour)
-
-	signer, err := NewGKESigner(s.ClusterSigningGKEKubeconfig, s.ClusterSigningGKERetryBackoff.Duration, recorder, client)
-	if err != nil {
-		return err
-	}
-
-	controller := certificates.NewCertificateController(
-		client,
-		sharedInformers.Certificates().V1beta1().CertificateSigningRequests(),
-		signer.handle,
-	)
-
-	sharedInformers.Start(nil)
-	controller.Run(5, nil) // runs forever
-	panic("unreachable")
-}

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/app/gke_signer.go

@@ -1,137 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package app
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/golang/glog"
-
-	capi "k8s.io/api/certificates/v1beta1"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/util/webhook"
-	clientset "k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/record"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
-	_ "k8s.io/kubernetes/pkg/apis/certificates/install"
-	"k8s.io/kubernetes/pkg/controller/certificates"
-)
-
-var (
-	groupVersions = []schema.GroupVersion{capi.SchemeGroupVersion}
-)
-
-// GKESigner uses external calls to GKE in order to sign certificate signing
-// requests.
-type GKESigner struct {
-	webhook        *webhook.GenericWebhook
-	kubeConfigFile string
-	retryBackoff   time.Duration
-	recorder       record.EventRecorder
-	client         clientset.Interface
-}
-
-// NewGKESigner will create a new instance of a GKESigner.
-func NewGKESigner(kubeConfigFile string, retryBackoff time.Duration, recorder record.EventRecorder, client clientset.Interface) (*GKESigner, error) {
-	webhook, err := webhook.NewGenericWebhook(legacyscheme.Registry, legacyscheme.Codecs, kubeConfigFile, groupVersions, retryBackoff)
-	if err != nil {
-		return nil, err
-	}
-
-	return &GKESigner{
-		webhook:        webhook,
-		kubeConfigFile: kubeConfigFile,
-		retryBackoff:   retryBackoff,
-		recorder:       recorder,
-		client:         client,
-	}, nil
-}
-
-func (s *GKESigner) handle(csr *capi.CertificateSigningRequest) error {
-	if !certificates.IsCertificateRequestApproved(csr) {
-		return nil
-	}
-	csr, err := s.sign(csr)
-	if err != nil {
-		return fmt.Errorf("error auto signing csr: %v", err)
-	}
-	_, err = s.client.CertificatesV1beta1().CertificateSigningRequests().UpdateStatus(csr)
-	if err != nil {
-		return fmt.Errorf("error updating signature for csr: %v", err)
-	}
-	return nil
-}
-
-// Sign will make an external call to GKE order to sign the given
-// *capi.CertificateSigningRequest, using the GKESigner's
-// kubeConfigFile.
-func (s *GKESigner) sign(csr *capi.CertificateSigningRequest) (*capi.CertificateSigningRequest, error) {
-	result := s.webhook.WithExponentialBackoff(func() rest.Result {
-		return s.webhook.RestClient.Post().Body(csr).Do()
-	})
-
-	if err := result.Error(); err != nil {
-		if bodyErr := s.resultBodyError(result); bodyErr != nil {
-			return nil, s.webhookError(csr, bodyErr)
-		}
-		return nil, s.webhookError(csr, err)
-	}
-
-	var statusCode int
-	if result.StatusCode(&statusCode); statusCode < 200 || statusCode >= 300 {
-		return nil, s.webhookError(csr, fmt.Errorf("received unsuccessful response code from webhook: %d", statusCode))
-	}
-
-	result_csr := &capi.CertificateSigningRequest{}
-
-	if err := result.Into(result_csr); err != nil {
-		return nil, s.webhookError(result_csr, err)
-	}
-
-	// Keep the original CSR intact, and only update fields we expect to change.
-	csr.Status.Certificate = result_csr.Status.Certificate
-	return csr, nil
-}
-
-func (s *GKESigner) webhookError(csr *capi.CertificateSigningRequest, err error) error {
-	glog.V(2).Infof("error contacting webhook backend: %s", err)
-	s.recorder.Eventf(csr, "Warning", "SigningError", "error while calling GKE: %v", err)
-	return err
-}
-
-// signResultError represents the structured response body of a failed call to
-// GKE's SignCertificate API.
-type signResultError struct {
-	Error struct {
-		Code    int
-		Message string
-		Status  string
-	}
-}
-
-// resultBodyError attempts to extract an error out of a response body.
-func (s *GKESigner) resultBodyError(result rest.Result) error {
-	body, _ := result.Raw()
-	var sre signResultError
-	if err := json.Unmarshal(body, &sre); err == nil {
-		return fmt.Errorf("server responded with error: %s", sre.Error.Message)
-	}
-	return nil
-}

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/app/gke_signer_test.go

@@ -1,154 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package app
-
-import (
-	"bytes"
-	"encoding/json"
-	"io/ioutil"
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"text/template"
-	"time"
-
-	certificates "k8s.io/api/certificates/v1beta1"
-	"k8s.io/client-go/tools/record"
-)
-
-const kubeConfigTmpl = `
-clusters:
-- cluster:
-    server: {{ .Server }}
-    name: testcluster
-users:
-- user:
-    username: admin
-    password: mypass
-`
-
-func TestGKESigner(t *testing.T) {
-	goodResponse := &certificates.CertificateSigningRequest{
-		Status: certificates.CertificateSigningRequestStatus{
-			Certificate: []byte("fake certificate"),
-		},
-	}
-
-	invalidResponse := "{ \"status\": \"Not a properly formatted CSR response\" }"
-
-	cases := []struct {
-		mockResponse interface{}
-		expected     []byte
-		failCalls    int
-		wantErr      bool
-	}{
-		{
-			mockResponse: goodResponse,
-			expected:     goodResponse.Status.Certificate,
-			wantErr:      false,
-		},
-		{
-			mockResponse: goodResponse,
-			expected:     goodResponse.Status.Certificate,
-			failCalls:    3,
-			wantErr:      false,
-		},
-		{
-			mockResponse: goodResponse,
-			failCalls:    20,
-			wantErr:      true,
-		},
-		{
-			mockResponse: invalidResponse,
-			wantErr:      true,
-		},
-	}
-
-	for _, c := range cases {
-		server, err := newTestServer(c.mockResponse, c.failCalls)
-		if err != nil {
-			t.Fatalf("error creating test server")
-		}
-
-		kubeConfig, err := ioutil.TempFile("", "kubeconfig")
-		if err != nil {
-			t.Fatalf("error creating kubeconfig tempfile: %v", err)
-		}
-
-		tmpl, err := template.New("kubeconfig").Parse(kubeConfigTmpl)
-		if err != nil {
-			t.Fatalf("error creating kubeconfig template: %v", err)
-		}
-
-		data := struct{ Server string }{server.httpserver.URL}
-
-		if err := tmpl.Execute(kubeConfig, data); err != nil {
-			t.Fatalf("error executing kubeconfig template: %v", err)
-		}
-
-		if err := kubeConfig.Close(); err != nil {
-			t.Fatalf("error closing kubeconfig template: %v", err)
-		}
-
-		signer, err := NewGKESigner(kubeConfig.Name(), time.Duration(500)*time.Millisecond, record.NewFakeRecorder(10), nil)
-		if err != nil {
-			t.Fatalf("error creating GKESigner: %v", err)
-		}
-
-		cert, err := signer.sign(&certificates.CertificateSigningRequest{})
-
-		if c.wantErr {
-			if err == nil {
-				t.Errorf("wanted error during GKE.Sign() call, got not none")
-			}
-		} else {
-			if err != nil {
-				t.Errorf("error while signing: %v", err)
-			}
-
-			if !bytes.Equal(cert.Status.Certificate, c.expected) {
-				t.Errorf("response certificate didn't match expected %v: %v", c.expected, cert)
-			}
-		}
-	}
-}
-
-type testServer struct {
-	httpserver *httptest.Server
-	failCalls  int
-	response   interface{}
-}
-
-func newTestServer(response interface{}, failCalls int) (*testServer, error) {
-	server := &testServer{
-		response:  response,
-		failCalls: failCalls,
-	}
-
-	server.httpserver = httptest.NewServer(server)
-	return server, nil
-}
-
-func (s *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if s.failCalls > 0 {
-		http.Error(w, "Service unavailable", 500)
-		s.failCalls--
-	} else {
-		w.Header().Set("Content-Type", "application/json")
-		json.NewEncoder(w).Encode(s.response)
-	}
-}

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/app/options.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package app implements a server that runs a stand-alone version of the
-// certificates controller.
-package app
-
-import (
-	"time"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
-	"github.com/spf13/pflag"
-)
-
-// GKECertificatesController is the main context object for the package.
-type GKECertificatesController struct {
-	Kubeconfig                    string
-	ClusterSigningGKEKubeconfig   string
-	ClusterSigningGKERetryBackoff metav1.Duration
-	ApproveAllKubeletCSRsForGroup string
-}
-
-// Create a new instance of a GKECertificatesController with default parameters.
-func NewGKECertificatesController() *GKECertificatesController {
-	s := &GKECertificatesController{
-		ClusterSigningGKERetryBackoff: metav1.Duration{Duration: 500 * time.Millisecond},
-	}
-	return s
-}
-
-// AddFlags adds flags for a specific GKECertificatesController to the
-// specified FlagSet.
-func (s *GKECertificatesController) AddFlags(fs *pflag.FlagSet) {
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-
-	fs.StringVar(&s.ClusterSigningGKEKubeconfig, "cluster-signing-gke-kubeconfig", s.ClusterSigningGKEKubeconfig, "If set, use the kubeconfig file to call GKE to sign cluster-scoped certificates instead of using a local private key.")
-	fs.DurationVar(&s.ClusterSigningGKERetryBackoff.Duration, "cluster-signing-gke-retry-backoff", s.ClusterSigningGKERetryBackoff.Duration, "The initial backoff to use when retrying requests to GKE. Additional attempts will use exponential backoff.")
-
-	fs.StringVar(&s.ApproveAllKubeletCSRsForGroup, "insecure-experimental-approve-all-kubelet-csrs-for-group", s.ApproveAllKubeletCSRsForGroup, "The group for which the controller-manager will auto approve all CSRs for kubelet client certificates.")
-}

vendor/k8s.io/kubernetes/cmd/gke-certificates-controller/main.go

@@ -1,49 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// The GKE certificates controller is responsible for monitoring certificate
-// signing requests and (potentially) auto-approving and signing them within
-// GKE.
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"k8s.io/apiserver/pkg/util/flag"
-	"k8s.io/kubernetes/cmd/gke-certificates-controller/app"
-	"k8s.io/kubernetes/pkg/kubectl/util/logs"
-	"k8s.io/kubernetes/pkg/version/verflag"
-
-	"github.com/spf13/pflag"
-)
-
-// TODO(pipejakob): Move this entire cmd directory into its own repo
-func main() {
-	s := app.NewGKECertificatesController()
-	s.AddFlags(pflag.CommandLine)
-
-	flag.InitFlags()
-	logs.InitLogs()
-	defer logs.FlushLogs()
-
-	verflag.PrintAndExitIfRequested()
-
-	if err := app.Run(s); err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
-	}
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/BUILD

@@ -4,69 +4,33 @@ load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_binary",
     "go_library",
-    "go_test",
 )
 load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "hyperkube",
-    importpath = "k8s.io/kubernetes/cmd/hyperkube",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     x_defs = version_x_defs(),
 )
 
-go_test(
-    name = "go_default_test",
-    srcs = ["hyperkube_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/hyperkube",
-    library = ":go_default_library",
-    deps = [
-        "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-    ],
-)
-
 go_library(
     name = "go_default_library",
-    srcs = [
-        "cloud-controller-manager.go",
-        "hyperkube.go",
-        "kube-aggregator.go",
-        "kube-apiserver.go",
-        "kube-controller-manager.go",
-        "kube-proxy.go",
-        "kube-scheduler.go",
-        "kubectl.go",
-        "kubelet.go",
-        "main.go",
-        "server.go",
-    ],
+    srcs = ["main.go"],
     importpath = "k8s.io/kubernetes/cmd/hyperkube",
     deps = [
         "//cmd/cloud-controller-manager/app:go_default_library",
-        "//cmd/cloud-controller-manager/app/options:go_default_library",
         "//cmd/kube-apiserver/app:go_default_library",
-        "//cmd/kube-apiserver/app/options:go_default_library",
         "//cmd/kube-controller-manager/app:go_default_library",
-        "//cmd/kube-controller-manager/app/options:go_default_library",
         "//cmd/kube-proxy/app:go_default_library",
+        "//cmd/kube-scheduler/app:go_default_library",
         "//cmd/kubelet/app:go_default_library",
-        "//cmd/kubelet/app/options:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
         "//pkg/kubectl/cmd:go_default_library",
-        "//pkg/kubectl/cmd/util:go_default_library",
-        "//pkg/util/template:go_default_library",
         "//pkg/version/prometheus:go_default_library",
-        "//pkg/version/verflag:go_default_library",
-        "//plugin/cmd/kube-scheduler/app:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
-        "//vendor/k8s.io/kube-aggregator/pkg/cmd/server:go_default_library",
     ],
 )
 

vendor/k8s.io/kubernetes/cmd/hyperkube/cloud-controller-manager.go

@@ -1,39 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"k8s.io/kubernetes/cmd/cloud-controller-manager/app"
-	"k8s.io/kubernetes/cmd/cloud-controller-manager/app/options"
-)
-
-// NewCloudControllerManager creates a new hyperkube Server object that includes the
-// description and flags.
-func NewCloudControllerManager() *Server {
-	s := options.NewCloudControllerManagerServer()
-
-	hks := Server{
-		name:        "cloud-controller-manager",
-		SimpleUsage: "cloud-controller-manager",
-		Long:        "A server that acts as an external cloud provider.",
-		Run: func(_ *Server, args []string, stopCh <-chan struct{}) error {
-			return app.Run(s)
-		},
-	}
-	s.AddFlags(hks.Flags())
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/hyperkube.go

@@ -1,281 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-
-	"github.com/spf13/pflag"
-
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/apiserver/pkg/server"
-	utilflag "k8s.io/apiserver/pkg/util/flag"
-	"k8s.io/apiserver/pkg/util/logs"
-	utiltemplate "k8s.io/kubernetes/pkg/util/template"
-	"k8s.io/kubernetes/pkg/version/verflag"
-)
-
-// HyperKube represents a single binary that can morph/manage into multiple
-// servers.
-type HyperKube struct {
-	Name string // The executable name, used for help and soft-link invocation
-	Long string // A long description of the binary.  It will be world wrapped before output.
-
-	servers             []Server
-	alphaServers        []Server
-	baseFlags           *pflag.FlagSet
-	out                 io.Writer
-	helpFlagVal         bool
-	makeSymlinksFlagVal bool
-}
-
-// AddServer adds a server to the HyperKube object.
-func (hk *HyperKube) AddServer(s *Server) {
-	hk.servers = append(hk.servers, *s)
-	hk.servers[len(hk.servers)-1].hk = hk
-}
-
-// AddServer adds an alpha server to the HyperKube object.
-func (hk *HyperKube) AddAlphaServer(s *Server) {
-	hk.alphaServers = append(hk.alphaServers, *s)
-	hk.alphaServers[len(hk.alphaServers)-1].hk = hk
-}
-
-// FindServer will find a specific server named name.
-func (hk *HyperKube) FindServer(name string) (*Server, error) {
-	return findServer(name, hk.servers)
-}
-
-// FindServer will find a specific alpha server named name.
-func (hk *HyperKube) FindAlphaServer(name string) (*Server, error) {
-	return findServer(name, hk.alphaServers)
-}
-
-func findServer(name string, servers []Server) (*Server, error) {
-	for _, s := range servers {
-		if s.Name() == name || s.AlternativeName == name {
-			return &s, nil
-		}
-	}
-	return nil, fmt.Errorf("Server not found: %s", name)
-}
-
-// Servers returns a list of all of the registered servers
-func (hk *HyperKube) Servers() []Server {
-	return hk.servers
-}
-
-// Flags returns a flagset for "global" flags.
-func (hk *HyperKube) Flags() *pflag.FlagSet {
-	if hk.baseFlags == nil {
-		hk.baseFlags = pflag.NewFlagSet(hk.Name, pflag.ContinueOnError)
-		hk.baseFlags.SetOutput(ioutil.Discard)
-		hk.baseFlags.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
-		hk.baseFlags.BoolVarP(&hk.helpFlagVal, "help", "h", false, "help for "+hk.Name)
-		hk.baseFlags.BoolVar(&hk.makeSymlinksFlagVal, "make-symlinks", false, "create a symlink for each server in current directory")
-		hk.baseFlags.MarkHidden("make-symlinks") // hide this flag from appearing in servers' usage output
-
-		// These will add all of the "global" flags (defined with both the
-		// flag and pflag packages) to the new flag set we have.
-		hk.baseFlags.AddGoFlagSet(flag.CommandLine)
-		hk.baseFlags.AddFlagSet(pflag.CommandLine)
-
-	}
-	return hk.baseFlags
-}
-
-// Out returns the io.Writer that is used for all usage/error information
-func (hk *HyperKube) Out() io.Writer {
-	if hk.out == nil {
-		hk.out = os.Stderr
-	}
-	return hk.out
-}
-
-// SetOut sets the output writer for all usage/error information
-func (hk *HyperKube) SetOut(w io.Writer) {
-	hk.out = w
-}
-
-// Print is a convenience method to Print to the defined output
-func (hk *HyperKube) Print(i ...interface{}) {
-	fmt.Fprint(hk.Out(), i...)
-}
-
-// Println is a convenience method to Println to the defined output
-func (hk *HyperKube) Println(i ...interface{}) {
-	fmt.Fprintln(hk.Out(), i...)
-}
-
-// Printf is a convenience method to Printf to the defined output
-func (hk *HyperKube) Printf(format string, i ...interface{}) {
-	fmt.Fprintf(hk.Out(), format, i...)
-}
-
-// Run the server.  This will pick the appropriate server and run it.
-func (hk *HyperKube) Run(args []string, stopCh <-chan struct{}) error {
-	// If we are called directly, parse all flags up to the first real
-	// argument.  That should be the server to run.
-	command := args[0]
-	serverName := path.Base(command)
-	args = args[1:]
-	if serverName == hk.Name {
-
-		baseFlags := hk.Flags()
-		baseFlags.SetInterspersed(false) // Only parse flags up to the next real command
-		err := baseFlags.Parse(args)
-		if err != nil || hk.helpFlagVal {
-			if err != nil {
-				hk.Println("Error:", err)
-			}
-			hk.Usage()
-			return err
-		}
-
-		if hk.makeSymlinksFlagVal {
-			return hk.MakeSymlinks(command)
-		}
-
-		verflag.PrintAndExitIfRequested()
-
-		args = baseFlags.Args()
-		if len(args) > 0 && len(args[0]) > 0 {
-			serverName = args[0]
-			args = args[1:]
-		} else {
-			err = errors.New("no server specified")
-			hk.Printf("Error: %v\n\n", err)
-			hk.Usage()
-			return err
-		}
-	}
-
-	var s *Server
-	var err error
-	if serverName == "alpha" {
-		if len(args) > 0 && len(args[0]) > 0 {
-			serverName = args[0]
-			args = args[1:]
-			hk.Printf("Warning: alpha command syntax is unstable!\n\n")
-		} else {
-			err = errors.New("no alpha server specified")
-			hk.Printf("Error: %v\n\n", err)
-			hk.Usage()
-			return err
-		}
-		s, err = hk.FindAlphaServer(serverName)
-	} else {
-		s, err = hk.FindServer(serverName)
-	}
-	if err != nil {
-		hk.Printf("Error: %v\n\n", err)
-		hk.Usage()
-		return err
-	}
-
-	s.Flags().AddFlagSet(hk.Flags())
-	err = s.Flags().Parse(args)
-	if err != nil || hk.helpFlagVal {
-		if err != nil {
-			hk.Printf("Error: %v\n\n", err)
-		}
-		s.Usage()
-		return err
-	}
-
-	verflag.PrintAndExitIfRequested()
-
-	logs.InitLogs()
-	defer logs.FlushLogs()
-
-	if !s.RespectsStopCh {
-		// For commands that do not respect the stopCh, we run them in a go
-		// routine and leave them running when stopCh is closed.
-		errCh := make(chan error)
-		go func() {
-			errCh <- s.Run(s, s.Flags().Args(), wait.NeverStop)
-		}()
-		select {
-		case <-stopCh:
-			return errors.New("interrupted") // This error text is ignored.
-		case err = <-errCh:
-			// fall-through
-		}
-	} else {
-		err = s.Run(s, s.Flags().Args(), stopCh)
-	}
-	if err != nil {
-		hk.Println("Error:", err)
-	}
-
-	return err
-}
-
-// RunToExit will run the hyperkube and then call os.Exit with an appropriate exit code.
-func (hk *HyperKube) RunToExit(args []string) {
-	stopCh := server.SetupSignalHandler()
-	if err := hk.Run(args, stopCh); err != nil {
-		os.Exit(1)
-	}
-}
-
-// Usage will write out a summary for all servers that this binary supports.
-func (hk *HyperKube) Usage() {
-	tt := `{{if .Long}}{{.Long | trim | wrap ""}}
-{{end}}Usage
-
-  {{.Name}} <server> [flags]
-
-Servers
-{{range .Servers}}
-  {{.Name}}
-{{.Long | trim | wrap "    "}}{{end}}
-Call '{{.Name}} --make-symlinks' to create symlinks for each server in the local directory.
-Call '{{.Name}} <server> --help' for help on a specific server.
-`
-	utiltemplate.ExecuteTemplate(hk.Out(), tt, hk)
-}
-
-// MakeSymlinks will create a symlink for each registered hyperkube server in the local directory.
-func (hk *HyperKube) MakeSymlinks(command string) error {
-	wd, err := os.Getwd()
-	if err != nil {
-		return err
-	}
-
-	var errs bool
-	for _, s := range hk.servers {
-		link := path.Join(wd, s.Name())
-
-		err := os.Symlink(command, link)
-		if err != nil {
-			errs = true
-			hk.Println(err)
-		}
-	}
-
-	if errs {
-		return errors.New("Error creating one or more symlinks.")
-	}
-	return nil
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/hyperkube_test.go

@@ -1,338 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/spf13/cobra"
-	"github.com/stretchr/testify/assert"
-
-	"k8s.io/apimachinery/pkg/util/wait"
-)
-
-type result struct {
-	err    error
-	output string
-}
-
-func testServer(n string) *Server {
-	return &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A simple server named %s", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			s.hk.Printf("%s Run\n", s.Name())
-			return nil
-		},
-	}
-}
-func testServerError(n string) *Server {
-	return &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A simple server named %s that returns an error", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			s.hk.Printf("%s Run\n", s.Name())
-			return errors.New("server returning error")
-		},
-	}
-}
-func testStopChRespectingServer(n string) *Server {
-	return &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A simple server named %s", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			s.hk.Printf("%s Run\n", s.Name())
-			<-stopCh
-			return nil
-		},
-		RespectsStopCh: true,
-	}
-}
-func testStopChIgnoringServer(n string) *Server {
-	return &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A simple server named %s", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			<-wait.NeverStop // this leaks obviously, but we don't care about one go routine more or less in test
-			return nil
-		},
-		RespectsStopCh: false,
-	}
-}
-func testStopChRespectingServerWithError(n string) *Server {
-	return &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A simple server named %s", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			s.hk.Printf("%s Run\n", s.Name())
-			<-stopCh
-			return errors.New("server returning error")
-		},
-		RespectsStopCh: true,
-	}
-}
-
-const defaultCobraMessage = "default message from cobra command"
-const defaultCobraSubMessage = "default sub-message from cobra command"
-const cobraMessageDesc = "message to print"
-const cobraSubMessageDesc = "sub-message to print"
-
-func testCobraCommand(n string) *Server {
-
-	var cobraServer *Server
-	var msg string
-	cmd := &cobra.Command{
-		Use:   n,
-		Long:  n,
-		Short: n,
-		Run: func(cmd *cobra.Command, args []string) {
-			cobraServer.hk.Printf("msg: %s\n", msg)
-		},
-	}
-	cmd.PersistentFlags().StringVar(&msg, "msg", defaultCobraMessage, cobraMessageDesc)
-
-	var subMsg string
-	subCmdName := "subcommand"
-	subCmd := &cobra.Command{
-		Use:   subCmdName,
-		Long:  subCmdName,
-		Short: subCmdName,
-		Run: func(cmd *cobra.Command, args []string) {
-			cobraServer.hk.Printf("submsg: %s", subMsg)
-		},
-	}
-	subCmd.PersistentFlags().StringVar(&subMsg, "submsg", defaultCobraSubMessage, cobraSubMessageDesc)
-
-	cmd.AddCommand(subCmd)
-
-	localFlags := cmd.LocalFlags()
-	localFlags.SetInterspersed(false)
-	s := &Server{
-		SimpleUsage: n,
-		Long:        fmt.Sprintf("A server named %s which uses a cobra command", n),
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			cobraServer = s
-			cmd.SetOutput(s.hk.Out())
-			cmd.SetArgs(args)
-			return cmd.Execute()
-		},
-		flags: localFlags,
-	}
-
-	return s
-}
-func runFull(t *testing.T, args string, stopCh <-chan struct{}) *result {
-	buf := new(bytes.Buffer)
-	hk := HyperKube{
-		Name: "hyperkube",
-		Long: "hyperkube is an all-in-one server binary.",
-	}
-	hk.SetOut(buf)
-
-	hk.AddServer(testServer("test1"))
-	hk.AddServer(testServer("test2"))
-	hk.AddServer(testServer("test3"))
-	hk.AddAlphaServer(testServer("testAlpha1"))
-	hk.AddServer(testServerError("test-error"))
-	hk.AddServer(testStopChIgnoringServer("test-stop-ch-ignoring"))
-	hk.AddServer(testStopChRespectingServer("test-stop-ch-respecting"))
-	hk.AddServer(testStopChRespectingServerWithError("test-error-stop-ch-respecting"))
-	hk.AddServer(testCobraCommand("test-cobra-command"))
-
-	a := strings.Split(args, " ")
-	t.Logf("Running full with args: %q", a)
-	err := hk.Run(a, stopCh)
-
-	r := &result{err, buf.String()}
-	t.Logf("Result err: %v, output: %q", r.err, r.output)
-
-	return r
-}
-
-func TestRun(t *testing.T) {
-	x := runFull(t, "hyperkube test1", wait.NeverStop)
-	assert.Contains(t, x.output, "test1 Run")
-	assert.NoError(t, x.err)
-}
-
-func TestLinkRun(t *testing.T) {
-	x := runFull(t, "test1", wait.NeverStop)
-	assert.Contains(t, x.output, "test1 Run")
-	assert.NoError(t, x.err)
-}
-
-func TestTopNoArgs(t *testing.T) {
-	x := runFull(t, "hyperkube", wait.NeverStop)
-	assert.EqualError(t, x.err, "no server specified")
-}
-
-func TestBadServer(t *testing.T) {
-	x := runFull(t, "hyperkube bad-server", wait.NeverStop)
-	assert.EqualError(t, x.err, "Server not found: bad-server")
-	assert.Contains(t, x.output, "Usage")
-}
-
-func TestTopHelp(t *testing.T) {
-	x := runFull(t, "hyperkube --help", wait.NeverStop)
-	assert.NoError(t, x.err)
-	assert.Contains(t, x.output, "all-in-one")
-	assert.Contains(t, x.output, "A simple server named test1")
-}
-
-func TestTopFlags(t *testing.T) {
-	x := runFull(t, "hyperkube --help test1", wait.NeverStop)
-	assert.NoError(t, x.err)
-	assert.Contains(t, x.output, "all-in-one")
-	assert.Contains(t, x.output, "A simple server named test1")
-	assert.NotContains(t, x.output, "test1 Run")
-}
-
-func TestTopFlagsBad(t *testing.T) {
-	x := runFull(t, "hyperkube --bad-flag", wait.NeverStop)
-	assert.EqualError(t, x.err, "unknown flag: --bad-flag")
-	assert.Contains(t, x.output, "all-in-one")
-	assert.Contains(t, x.output, "A simple server named test1")
-}
-
-func TestServerHelp(t *testing.T) {
-	x := runFull(t, "hyperkube test1 --help", wait.NeverStop)
-	assert.NoError(t, x.err)
-	assert.Contains(t, x.output, "A simple server named test1")
-	assert.Contains(t, x.output, "-h, --help")
-	assert.Contains(t, x.output, "help for hyperkube")
-	assert.NotContains(t, x.output, "test1 Run")
-}
-
-func TestServerFlagsBad(t *testing.T) {
-	x := runFull(t, "hyperkube test1 --bad-flag", wait.NeverStop)
-	assert.EqualError(t, x.err, "unknown flag: --bad-flag")
-	assert.Contains(t, x.output, "A simple server named test1")
-	assert.Contains(t, x.output, "-h, --help")
-	assert.Contains(t, x.output, "help for hyperkube")
-	assert.NotContains(t, x.output, "test1 Run")
-}
-
-func TestServerError(t *testing.T) {
-	x := runFull(t, "hyperkube test-error", wait.NeverStop)
-	assert.Contains(t, x.output, "test-error Run")
-	assert.EqualError(t, x.err, "server returning error")
-}
-
-func TestAlphaRun(t *testing.T) {
-	x := runFull(t, "hyperkube alpha testAlpha1", wait.NeverStop)
-	assert.NoError(t, x.err)
-}
-
-func TestAlphaNoArgs(t *testing.T) {
-	x := runFull(t, "hyperkube alpha", wait.NeverStop)
-	assert.EqualError(t, x.err, "no alpha server specified")
-}
-
-func TestAlphaBadServer(t *testing.T) {
-	x := runFull(t, "hyperkube alpha bad-server", wait.NeverStop)
-	assert.EqualError(t, x.err, "Server not found: bad-server")
-	assert.Contains(t, x.output, "Usage")
-}
-
-func TestStopChIgnoringServer(t *testing.T) {
-	stopCh := make(chan struct{})
-	returnedCh := make(chan struct{})
-	var x *result
-	go func() {
-		defer close(returnedCh)
-		x = runFull(t, "hyperkube test-stop-ch-ignoring", stopCh)
-	}()
-	close(stopCh)
-	select {
-	case <-time.After(wait.ForeverTestTimeout):
-		t.Fatalf("%q never returned after stopCh was closed", "hyperkube test-stop-ch-ignoring")
-	case <-returnedCh:
-	}
-	// we cannot be sure that the server had a chance to output anything
-	// assert.Contains(t, x.output, "test-error-stop-ch-ignoring Run")
-	assert.EqualError(t, x.err, "interrupted")
-}
-
-func TestStopChRespectingServer(t *testing.T) {
-	stopCh := make(chan struct{})
-	returnedCh := make(chan struct{})
-	var x *result
-	go func() {
-		defer close(returnedCh)
-		x = runFull(t, "hyperkube test-stop-ch-respecting", stopCh)
-	}()
-	close(stopCh)
-	select {
-	case <-time.After(wait.ForeverTestTimeout):
-		t.Fatalf("%q never returned after stopCh was closed", "hyperkube test-stop-ch-respecting")
-	case <-returnedCh:
-	}
-	assert.Contains(t, x.output, "test-stop-ch-respecting Run")
-	assert.Nil(t, x.err)
-}
-
-func TestStopChRespectingServerWithError(t *testing.T) {
-	stopCh := make(chan struct{})
-	returnedCh := make(chan struct{})
-	var x *result
-	go func() {
-		defer close(returnedCh)
-		x = runFull(t, "hyperkube test-error-stop-ch-respecting", stopCh)
-	}()
-	close(stopCh)
-	select {
-	case <-time.After(wait.ForeverTestTimeout):
-		t.Fatalf("%q never returned after stopCh was closed", "hyperkube test-error-stop-ch-respecting")
-	case <-returnedCh:
-	}
-	assert.Contains(t, x.output, "test-error-stop-ch-respecting Run")
-	assert.EqualError(t, x.err, "server returning error")
-}
-
-func TestCobraCommandHelp(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command --help", wait.NeverStop)
-	assert.NoError(t, x.err)
-	assert.Contains(t, x.output, "A server named test-cobra-command which uses a cobra command")
-	assert.Contains(t, x.output, cobraMessageDesc)
-}
-func TestCobraCommandDefaultMessage(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command", wait.NeverStop)
-	assert.Contains(t, x.output, fmt.Sprintf("msg: %s", defaultCobraMessage))
-}
-func TestCobraCommandMessage(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command --msg foobar", wait.NeverStop)
-	assert.Contains(t, x.output, "msg: foobar")
-}
-
-func TestCobraSubCommandHelp(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command subcommand --help", wait.NeverStop)
-	assert.NoError(t, x.err)
-	assert.Contains(t, x.output, cobraSubMessageDesc)
-}
-func TestCobraSubCommandDefaultMessage(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command subcommand", wait.NeverStop)
-	assert.Contains(t, x.output, fmt.Sprintf("submsg: %s", defaultCobraSubMessage))
-}
-func TestCobraSubCommandMessage(t *testing.T) {
-	x := runFull(t, "hyperkube test-cobra-command subcommand --submsg foobar", wait.NeverStop)
-	assert.Contains(t, x.output, "submsg: foobar")
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kube-aggregator.go

@@ -1,52 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"os"
-
-	"k8s.io/kube-aggregator/pkg/cmd/server"
-)
-
-// NewKubeAggregator creates a new hyperkube Server object that includes the
-// description and flags.
-func NewKubeAggregator() *Server {
-	o := server.NewDefaultOptions(os.Stdout, os.Stderr)
-
-	hks := Server{
-		name:            "aggregator",
-		AlternativeName: "kube-aggregator",
-		SimpleUsage:     "aggregator",
-		Long:            "Aggregator for Kubernetes-style API servers: dynamic registration, discovery summarization, secure proxy.",
-		Run: func(_ *Server, args []string, stopCh <-chan struct{}) error {
-			if err := o.Complete(); err != nil {
-				return err
-			}
-			if err := o.Validate(args); err != nil {
-				return err
-			}
-			if err := o.RunAggregator(stopCh); err != nil {
-				return err
-			}
-			return nil
-		},
-		RespectsStopCh: true,
-	}
-
-	o.AddFlags(hks.Flags())
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kube-apiserver.go

@@ -1,41 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"k8s.io/kubernetes/cmd/kube-apiserver/app"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
-)
-
-// NewKubeAPIServer creates a new hyperkube Server object that includes the
-// description and flags.
-func NewKubeAPIServer() *Server {
-	s := options.NewServerRunOptions()
-
-	hks := Server{
-		name:            "apiserver",
-		AlternativeName: "kube-apiserver",
-		SimpleUsage:     "apiserver",
-		Long:            "The main API entrypoint and interface to the storage system.  The API server is also the focal point for all authorization decisions.",
-		Run: func(_ *Server, args []string, stopCh <-chan struct{}) error {
-			return app.Run(s, stopCh)
-		},
-		RespectsStopCh: true,
-	}
-	s.AddFlags(hks.Flags())
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kube-controller-manager.go

@@ -1,40 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"k8s.io/kubernetes/cmd/kube-controller-manager/app"
-	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
-)
-
-// NewKubeControllerManager creates a new hyperkube Server object that includes the
-// description and flags.
-func NewKubeControllerManager() *Server {
-	s := options.NewCMServer()
-
-	hks := Server{
-		name:            "controller-manager",
-		AlternativeName: "kube-controller-manager",
-		SimpleUsage:     "controller-manager",
-		Long:            "A server that runs a set of active components. This includes replication controllers, service endpoints and nodes.",
-		Run: func(_ *Server, args []string, stopCh <-chan struct{}) error {
-			return app.Run(s)
-		},
-	}
-	s.AddFlags(hks.Flags(), app.KnownControllers(), app.ControllersDisabledByDefault.List())
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kube-proxy.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"flag"
-
-	"k8s.io/apiserver/pkg/server/healthz"
-	"k8s.io/kubernetes/cmd/kube-proxy/app"
-)
-
-// NewKubeProxy creates a new hyperkube Server object that includes the
-// description and flags.
-func NewKubeProxy() *Server {
-	healthz.DefaultHealthz()
-
-	command := app.NewProxyCommand()
-
-	hks := Server{
-		name:            "proxy",
-		AlternativeName: "kube-proxy",
-		SimpleUsage:     "proxy",
-		Long:            command.Long,
-	}
-
-	serverFlags := hks.Flags()
-	serverFlags.AddFlagSet(command.Flags())
-
-	// FIXME this is here because hyperkube does its own flag parsing, and we need
-	// the command to know about the go flag set. Remove this once hyperkube is
-	// refactored to use cobra throughout.
-	command.Flags().AddGoFlagSet(flag.CommandLine)
-
-	hks.Run = func(_ *Server, args []string, stopCh <-chan struct{}) error {
-		command.SetArgs(args)
-		return command.Execute()
-	}
-
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kube-scheduler.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"flag"
-
-	"k8s.io/apiserver/pkg/server/healthz"
-	"k8s.io/kubernetes/plugin/cmd/kube-scheduler/app"
-)
-
-// NewScheduler creates a new hyperkube Server object that includes the
-// description and flags.
-func NewScheduler() *Server {
-	healthz.DefaultHealthz()
-
-	command := app.NewSchedulerCommand()
-
-	hks := Server{
-		name:            "scheduler",
-		AlternativeName: "kube-scheduler",
-		SimpleUsage:     "scheduler",
-		Long:            command.Long,
-	}
-
-	serverFlags := hks.Flags()
-	serverFlags.AddFlagSet(command.Flags())
-
-	// FIXME this is here because hyperkube does its own flag parsing, and we need
-	// the command to know about the go flag set. Remove this once hyperkube is
-	// refactored to use cobra throughout.
-	command.Flags().AddGoFlagSet(flag.CommandLine)
-
-	hks.Run = func(_ *Server, args []string, stopCh <-chan struct{}) error {
-		command.SetArgs(args)
-		return command.Execute()
-	}
-
-	return &hks
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kubectl.go

@@ -1,42 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"os"
-
-	"k8s.io/kubernetes/pkg/kubectl/cmd"
-	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
-)
-
-// Create a Server that implements the kubectl command
-func NewKubectlServer() *Server {
-	cmd := cmd.NewKubectlCommand(cmdutil.NewFactory(nil), os.Stdin, os.Stdout, os.Stderr)
-	localFlags := cmd.LocalFlags()
-	localFlags.SetInterspersed(false)
-
-	return &Server{
-		name:        "kubectl",
-		SimpleUsage: "Kubernetes command line client",
-		Long:        "Kubernetes command line client",
-		Run: func(s *Server, args []string, stopCh <-chan struct{}) error {
-			cmd.SetArgs(args)
-			return cmd.Execute()
-		},
-		flags: localFlags,
-	}
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/kubelet.go

@@ -1,50 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"k8s.io/kubernetes/cmd/kubelet/app"
-	"k8s.io/kubernetes/cmd/kubelet/app/options"
-)
-
-// NewKubelet creates a new hyperkube Server object that includes the
-// description and flags.
-func NewKubelet() (*Server, error) {
-	s, err := options.NewKubeletServer()
-	if err != nil {
-		return nil, err
-	}
-
-	hks := Server{
-		name:        "kubelet",
-		SimpleUsage: "kubelet",
-		Long: `The kubelet binary is responsible for maintaining a set of containers on a
-		particular node. It syncs data from a variety of sources including a
-		Kubernetes API server, an etcd cluster, HTTP endpoint or local file. It then
-		queries Docker to see what is currently running.  It synchronizes the
-		configuration data, with the running set of containers by starting or stopping
-		Docker containers.`,
-		Run: func(_ *Server, _ []string, stopCh <-chan struct{}) error {
-			if s.ExperimentalDockershim {
-				return app.RunDockershim(&s.KubeletFlags, &s.KubeletConfiguration)
-			}
-			return app.Run(s, nil)
-		},
-	}
-	s.AddFlags(hks.Flags())
-	return &hks, nil
-}

vendor/k8s.io/kubernetes/cmd/hyperkube/main.go

@@ -20,34 +20,156 @@ limitations under the License.
 package main
 
 import (
+	"errors"
+	goflag "flag"
 	"fmt"
+	"math/rand"
 	"os"
+	"path"
+	"path/filepath"
+	"time"
 
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+
+	utilflag "k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
+	cloudcontrollermanager "k8s.io/kubernetes/cmd/cloud-controller-manager/app"
+	kubeapiserver "k8s.io/kubernetes/cmd/kube-apiserver/app"
+	kubecontrollermanager "k8s.io/kubernetes/cmd/kube-controller-manager/app"
+	kubeproxy "k8s.io/kubernetes/cmd/kube-proxy/app"
+	kubescheduler "k8s.io/kubernetes/cmd/kube-scheduler/app"
+	kubelet "k8s.io/kubernetes/cmd/kubelet/app"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
-	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+	kubectl "k8s.io/kubernetes/pkg/kubectl/cmd"
+	_ "k8s.io/kubernetes/pkg/version/prometheus" // for version metric registration
 )
 
 func main() {
-	hk := HyperKube{
-		Name: "hyperkube",
-		Long: "This is an all-in-one binary that can run any of the various Kubernetes servers.",
-	}
+	rand.Seed(time.Now().UTC().UnixNano())
 
-	hk.AddServer(NewKubectlServer())
-	hk.AddServer(NewKubeAPIServer())
-	hk.AddServer(NewKubeControllerManager())
-	hk.AddServer(NewScheduler())
-	if kubelet, err := NewKubelet(); err != nil {
-		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+	hyperkubeCommand, allCommandFns := NewHyperKubeCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	basename := filepath.Base(os.Args[0])
+	if err := commandFor(basename, hyperkubeCommand, allCommandFns).Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
-	} else {
-		hk.AddServer(kubelet)
 	}
-	hk.AddServer(NewKubeProxy())
-	hk.AddServer(NewKubeAggregator())
-
-	// Alpha servers
-	hk.AddAlphaServer(NewCloudControllerManager())
-
-	hk.RunToExit(os.Args)
+}
+
+func commandFor(basename string, defaultCommand *cobra.Command, commands []func() *cobra.Command) *cobra.Command {
+	for _, commandFn := range commands {
+		command := commandFn()
+		if command.Name() == basename {
+			return command
+		}
+		for _, alias := range command.Aliases {
+			if alias == basename {
+				return command
+			}
+		}
+	}
+
+	return defaultCommand
+}
+
+// NewCmdRequestProject implement the OpenShift cli RequestProject command.
+func NewHyperKubeCommand() (*cobra.Command, []func() *cobra.Command) {
+	// these have to be functions since the command is polymorphic.  Cobra wants you to be  top level
+	// command to get executed
+	apiserver := func() *cobra.Command {
+		ret := kubeapiserver.NewAPIServerCommand()
+		// add back some unfortunate aliases that should be removed
+		ret.Aliases = []string{"apiserver"}
+		return ret
+	}
+	controller := func() *cobra.Command {
+		ret := kubecontrollermanager.NewControllerManagerCommand()
+		// add back some unfortunate aliases that should be removed
+		ret.Aliases = []string{"controller-manager"}
+		return ret
+	}
+	proxy := func() *cobra.Command {
+		ret := kubeproxy.NewProxyCommand()
+		// add back some unfortunate aliases that should be removed
+		ret.Aliases = []string{"proxy"}
+		return ret
+	}
+	scheduler := func() *cobra.Command {
+		ret := kubescheduler.NewSchedulerCommand()
+		// add back some unfortunate aliases that should be removed
+		ret.Aliases = []string{"scheduler"}
+		return ret
+	}
+	kubectlCmd := func() *cobra.Command { return kubectl.NewDefaultKubectlCommand() }
+	kubelet := func() *cobra.Command { return kubelet.NewKubeletCommand() }
+	cloudController := func() *cobra.Command { return cloudcontrollermanager.NewCloudControllerManagerCommand() }
+
+	commandFns := []func() *cobra.Command{
+		apiserver,
+		controller,
+		proxy,
+		scheduler,
+		kubectlCmd,
+		kubelet,
+		cloudController,
+	}
+
+	makeSymlinksFlag := false
+	cmd := &cobra.Command{
+		Use:   "hyperkube",
+		Short: "Request a new project",
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) != 0 || !makeSymlinksFlag {
+				cmd.Help()
+				os.Exit(1)
+			}
+
+			if err := makeSymlinks(os.Args[0], commandFns); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err.Error())
+			}
+		},
+	}
+	cmd.Flags().BoolVar(&makeSymlinksFlag, "make-symlinks", makeSymlinksFlag, "create a symlink for each server in current directory")
+	cmd.Flags().MarkHidden("make-symlinks") // hide this flag from appearing in servers' usage output
+
+	for i := range commandFns {
+		cmd.AddCommand(commandFns[i]())
+	}
+
+	return cmd, commandFns
+}
+
+// makeSymlinks will create a symlink for each command in the local directory.
+func makeSymlinks(targetName string, commandFns []func() *cobra.Command) error {
+	wd, err := os.Getwd()
+	if err != nil {
+		return err
+	}
+
+	var errs bool
+	for _, commandFn := range commandFns {
+		command := commandFn()
+		link := path.Join(wd, command.Name())
+
+		err := os.Symlink(targetName, link)
+		if err != nil {
+			errs = true
+			fmt.Println(err)
+		}
+	}
+
+	if errs {
+		return errors.New("Error creating one or more symlinks.")
+	}
+	return nil
 }

vendor/k8s.io/kubernetes/cmd/hyperkube/server.go

@@ -1,77 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"io/ioutil"
-	"strings"
-
-	"k8s.io/apiserver/pkg/util/flag"
-	utiltemplate "k8s.io/kubernetes/pkg/util/template"
-
-	"github.com/spf13/pflag"
-)
-
-type serverRunFunc func(s *Server, args []string, stopCh <-chan struct{}) error
-
-// Server describes a server that this binary can morph into.
-type Server struct {
-	SimpleUsage     string        // One line description of the server.
-	Long            string        // Longer free form description of the server
-	Run             serverRunFunc // Run the server.  This is not expected to return.
-	AlternativeName string
-	RespectsStopCh  bool
-
-	flags *pflag.FlagSet // Flags for the command (and all dependents)
-	name  string
-	hk    *HyperKube
-}
-
-// Usage returns the full usage string including all of the flags.
-func (s *Server) Usage() error {
-	tt := `{{if .Long}}{{.Long | trim | wrap ""}}
-{{end}}Usage:
-  {{.SimpleUsage}} [flags]
-
-Available Flags:
-{{.Flags.FlagUsages}}`
-
-	return utiltemplate.ExecuteTemplate(s.hk.Out(), tt, s)
-}
-
-// Name returns the name of the command as derived from the usage line.
-func (s *Server) Name() string {
-	if s.name != "" {
-		return s.name
-	}
-	name := s.SimpleUsage
-	i := strings.Index(name, " ")
-	if i >= 0 {
-		name = name[:i]
-	}
-	return name
-}
-
-// Flags returns a flagset for this server
-func (s *Server) Flags() *pflag.FlagSet {
-	if s.flags == nil {
-		s.flags = pflag.NewFlagSet(s.Name(), pflag.ContinueOnError)
-		s.flags.SetOutput(ioutil.Discard)
-		s.flags.SetNormalizeFunc(flag.WordSepNormalizeFunc)
-	}
-	return s.flags
-}

vendor/k8s.io/kubernetes/cmd/importverifier/BUILD

@@ -8,8 +8,7 @@ load(
 
 go_binary(
     name = "importverifier",
-    importpath = "k8s.io/kubernetes/cmd/importverifier",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
 )
 
 go_library(

vendor/k8s.io/kubernetes/cmd/kube-apiserver/BUILD

@@ -9,14 +9,8 @@ load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "kube-apiserver",
-    gc_linkopts = [
-        "-linkmode",
-        "external",
-        "-extldflags",
-        "-static",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
+    pure = "on",
     x_defs = version_x_defs(),
 )
 
@@ -26,12 +20,9 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-apiserver",
     deps = [
         "//cmd/kube-apiserver/app:go_default_library",
-        "//cmd/kube-apiserver/app/options:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
         "//pkg/version/prometheus:go_default_library",
-        "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],

vendor/k8s.io/kubernetes/cmd/kube-apiserver/apiserver.go

@@ -19,6 +19,7 @@ limitations under the License.
 package main
 
 import (
+	goflag "flag"
 	"fmt"
 	"math/rand"
 	"os"
@@ -26,31 +27,29 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"k8s.io/apiserver/pkg/server"
-	"k8s.io/apiserver/pkg/util/flag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app"
-	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
 	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
-	"k8s.io/kubernetes/pkg/version/verflag"
 )
 
 func main() {
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	s := options.NewServerRunOptions()
-	s.AddFlags(pflag.CommandLine)
+	command := app.NewAPIServerCommand()
 
-	flag.InitFlags()
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	verflag.PrintAndExitIfRequested()
-
-	stopCh := server.SetupSignalHandler()
-	if err := app.Run(s, stopCh); err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
+	if err := command.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
 		os.Exit(1)
 	}
 }

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/BUILD

@@ -23,12 +23,14 @@ go_library(
         "//pkg/apis/events:go_default_library",
         "//pkg/apis/extensions:go_default_library",
         "//pkg/apis/networking:go_default_library",
+        "//pkg/apis/policy:go_default_library",
         "//pkg/apis/storage:go_default_library",
         "//pkg/capabilities:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
         "//pkg/cloudprovider:go_default_library",
         "//pkg/controller/serviceaccount:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/generated/openapi:go_default_library",
         "//pkg/kubeapiserver:go_default_library",
         "//pkg/kubeapiserver/admission:go_default_library",
@@ -43,14 +45,16 @@ go_library(
         "//pkg/quota/install:go_default_library",
         "//pkg/registry/cachesize:go_default_library",
         "//pkg/registry/rbac/rest:go_default_library",
+        "//pkg/serviceaccount:go_default_library",
+        "//pkg/util/flag:go_default_library",
         "//pkg/util/reflector/prometheus:go_default_library",
         "//pkg/util/workqueue/prometheus:go_default_library",
         "//pkg/version:go_default_library",
+        "//pkg/version/verflag:go_default_library",
         "//plugin/pkg/auth/authenticator/token/bootstrap:go_default_library",
         "//vendor/github.com/go-openapi/spec:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library",
         "//vendor/k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion:go_default_library",
@@ -73,13 +77,17 @@ go_library(
         "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/tools/cache:go_default_library",
+        "//vendor/k8s.io/client-go/util/cert:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",
+        "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/apiserver:go_default_library",
+        "//vendor/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/client/informers/internalversion/apiregistration/internalversion:go_default_library",
         "//vendor/k8s.io/kube-aggregator/pkg/controllers/autoregister:go_default_library",

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/aggregator.go

@@ -38,8 +38,10 @@ import (
 	kubeexternalinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/kube-aggregator/pkg/apis/apiregistration"
+	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
 	"k8s.io/kube-aggregator/pkg/apis/apiregistration/v1beta1"
 	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
 	apiregistrationclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/internalclientset/typed/apiregistration/internalversion"
 	informers "k8s.io/kube-aggregator/pkg/client/informers/internalversion/apiregistration/internalversion"
 	"k8s.io/kube-aggregator/pkg/controllers/autoregister"
@@ -58,9 +60,17 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command
 
 	// copy the etcd options so we don't mutate originals.
 	etcdOptions := *commandOptions.Etcd
-	etcdOptions.StorageConfig.Codec = aggregatorapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)
+	etcdOptions.StorageConfig.Codec = aggregatorscheme.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion, v1.SchemeGroupVersion)
 	genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
 
+	// override MergedResourceConfig with aggregator defaults and registry
+	if err := commandOptions.APIEnablement.ApplyTo(
+		&genericConfig,
+		aggregatorapiserver.DefaultAPIResourceConfigSource(),
+		aggregatorscheme.Registry); err != nil {
+		return nil, err
+	}
+
 	var err error
 	var certBytes, keyBytes []byte
 	if len(commandOptions.ProxyClientCertFile) > 0 && len(commandOptions.ProxyClientKeyFile) > 0 {
@@ -187,8 +197,12 @@ func makeAPIServiceAvailableHealthzCheck(name string, apiServices []*apiregistra
 	})
 }
 
+// priority defines group priority that is used in discovery. This controls
+// group position in the kubectl output.
 type priority struct {
-	group   int32
+	// group indicates the order of the group relative to other groups.
+	group int32
+	// version indicates the relative order of the version inside of its group.
 	version int32
 }
 
@@ -226,9 +240,13 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
 	{Group: "storage.k8s.io", Version: "v1beta1"}:                {group: 16800, version: 9},
 	{Group: "storage.k8s.io", Version: "v1alpha1"}:               {group: 16800, version: 1},
 	{Group: "apiextensions.k8s.io", Version: "v1beta1"}:          {group: 16700, version: 9},
+	{Group: "admissionregistration.k8s.io", Version: "v1"}:       {group: 16700, version: 15},
 	{Group: "admissionregistration.k8s.io", Version: "v1beta1"}:  {group: 16700, version: 12},
 	{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9},
 	{Group: "scheduling.k8s.io", Version: "v1alpha1"}:            {group: 16600, version: 9},
+	// Append a new group to the end of the list if unsure.
+	// You can use min(existing group)-100 as the initial value for a group.
+	// Version can be set to 9 (to have space around) for a new group.
 }
 
 func apiServicesToRegister(delegateAPIServer genericapiserver.DelegationTarget, registration autoregister.AutoAPIServiceRegistration) []*apiregistration.APIService {

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/apiextensions.go

@@ -39,6 +39,14 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, exte
 	etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)
 	genericConfig.RESTOptionsGetter = &genericoptions.SimpleRestOptionsFactory{Options: etcdOptions}
 
+	// override MergedResourceConfig with apiextensions defaults and registry
+	if err := commandOptions.APIEnablement.ApplyTo(
+		&genericConfig,
+		apiextensionsapiserver.DefaultAPIResourceConfigSource(),
+		apiextensionsapiserver.Registry); err != nil {
+		return nil, err
+	}
+
 	apiextensionsConfig := &apiextensionsapiserver.Config{
 		GenericConfig: &genericapiserver.RecommendedConfig{
 			Config:                genericConfig,

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/options/BUILD

@@ -10,59 +10,31 @@ go_library(
     name = "go_default_library",
     srcs = [
         "options.go",
-        "plugins.go",
         "validation.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/options",
     deps = [
+        "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/core:go_default_library",
         "//pkg/apis/core/validation:go_default_library",
-        "//pkg/cloudprovider/providers:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/kubeapiserver/options:go_default_library",
         "//pkg/kubelet/client:go_default_library",
         "//pkg/master/ports:go_default_library",
         "//pkg/master/reconcilers:go_default_library",
-        "//plugin/pkg/admission/admit:go_default_library",
-        "//plugin/pkg/admission/alwayspullimages:go_default_library",
-        "//plugin/pkg/admission/antiaffinity:go_default_library",
-        "//plugin/pkg/admission/defaulttolerationseconds:go_default_library",
-        "//plugin/pkg/admission/deny:go_default_library",
-        "//plugin/pkg/admission/eventratelimit:go_default_library",
-        "//plugin/pkg/admission/exec:go_default_library",
-        "//plugin/pkg/admission/extendedresourcetoleration:go_default_library",
-        "//plugin/pkg/admission/gc:go_default_library",
-        "//plugin/pkg/admission/imagepolicy:go_default_library",
-        "//plugin/pkg/admission/initialresources:go_default_library",
-        "//plugin/pkg/admission/limitranger:go_default_library",
-        "//plugin/pkg/admission/namespace/autoprovision:go_default_library",
-        "//plugin/pkg/admission/namespace/exists:go_default_library",
-        "//plugin/pkg/admission/noderestriction:go_default_library",
-        "//plugin/pkg/admission/persistentvolume/label:go_default_library",
-        "//plugin/pkg/admission/persistentvolume/resize:go_default_library",
-        "//plugin/pkg/admission/persistentvolumeclaim/pvcprotection:go_default_library",
-        "//plugin/pkg/admission/podnodeselector:go_default_library",
-        "//plugin/pkg/admission/podpreset:go_default_library",
-        "//plugin/pkg/admission/podtolerationrestriction:go_default_library",
-        "//plugin/pkg/admission/priority:go_default_library",
-        "//plugin/pkg/admission/resourcequota:go_default_library",
-        "//plugin/pkg/admission/security/podsecuritypolicy:go_default_library",
-        "//plugin/pkg/admission/securitycontext/scdeny:go_default_library",
-        "//plugin/pkg/admission/serviceaccount:go_default_library",
-        "//plugin/pkg/admission/storageclass/setdefault:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/apiextensions-apiserver/pkg/apiserver:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/admission:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
+        "//vendor/k8s.io/kube-aggregator/pkg/apiserver/scheme:go_default_library",
     ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = ["options_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/options",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     deps = [
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/core:go_default_library",

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/options/options.go

@@ -42,16 +42,16 @@ import (
 type ServerRunOptions struct {
 	GenericServerRunOptions *genericoptions.ServerRunOptions
 	Etcd                    *genericoptions.EtcdOptions
-	SecureServing           *genericoptions.SecureServingOptions
+	SecureServing           *genericoptions.SecureServingOptionsWithLoopback
 	InsecureServing         *kubeoptions.InsecureServingOptions
 	Audit                   *genericoptions.AuditOptions
 	Features                *genericoptions.FeatureOptions
-	Admission               *genericoptions.AdmissionOptions
+	Admission               *kubeoptions.AdmissionOptions
 	Authentication          *kubeoptions.BuiltInAuthenticationOptions
 	Authorization           *kubeoptions.BuiltInAuthorizationOptions
 	CloudProvider           *kubeoptions.CloudProviderOptions
 	StorageSerialization    *kubeoptions.StorageSerializationOptions
-	APIEnablement           *kubeoptions.APIEnablementOptions
+	APIEnablement           *genericoptions.APIEnablementOptions
 
 	AllowPrivileged           bool
 	EnableLogsHandler         bool
@@ -71,6 +71,8 @@ type ServerRunOptions struct {
 
 	MasterCount            int
 	EndpointReconcilerType string
+
+	ServiceAccountSigningKeyFile string
 }
 
 // NewServerRunOptions creates a new ServerRunOptions object with default parameters
@@ -82,12 +84,12 @@ func NewServerRunOptions() *ServerRunOptions {
 		InsecureServing:      kubeoptions.NewInsecureServingOptions(),
 		Audit:                genericoptions.NewAuditOptions(),
 		Features:             genericoptions.NewFeatureOptions(),
-		Admission:            genericoptions.NewAdmissionOptions(),
+		Admission:            kubeoptions.NewAdmissionOptions(),
 		Authentication:       kubeoptions.NewBuiltInAuthenticationOptions().WithAll(),
 		Authorization:        kubeoptions.NewBuiltInAuthorizationOptions(),
 		CloudProvider:        kubeoptions.NewCloudProviderOptions(),
 		StorageSerialization: kubeoptions.NewStorageSerializationOptions(),
-		APIEnablement:        kubeoptions.NewAPIEnablementOptions(),
+		APIEnablement:        genericoptions.NewAPIEnablementOptions(),
 
 		EnableLogsHandler:      true,
 		EventTTL:               1 * time.Hour,
@@ -113,13 +115,11 @@ func NewServerRunOptions() *ServerRunOptions {
 		},
 		ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
 	}
+	s.ServiceClusterIPRange = kubeoptions.DefaultServiceIPCIDR
+
 	// Overwrite the default for storage data format.
 	s.Etcd.DefaultStorageMediaType = "application/vnd.kubernetes.protobuf"
 
-	// register all admission plugins
-	RegisterAllAdmissionPlugins(s.Admission.Plugins)
-	// Set the default for admission plugins names
-	s.Admission.PluginNames = []string{"AlwaysAdmit"}
 	return &s
 }
 
@@ -129,7 +129,6 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 	s.GenericServerRunOptions.AddUniversalFlags(fs)
 	s.Etcd.AddFlags(fs)
 	s.SecureServing.AddFlags(fs)
-	s.SecureServing.AddDeprecatedFlags(fs)
 	s.InsecureServing.AddFlags(fs)
 	s.InsecureServing.AddDeprecatedFlags(fs)
 	s.Audit.AddFlags(fs)
@@ -226,7 +225,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 		"api-server and calling out to webhook admission plugins. It is expected that this "+
 		"cert includes a signature from the CA in the --requestheader-client-ca-file flag. "+
 		"That CA is published in the 'extension-apiserver-authentication' configmap in "+
-		"the kube-system namespace. Components recieving calls from kube-aggregator should "+
+		"the kube-system namespace. Components receiving calls from kube-aggregator should "+
 		"use that CA to perform their half of the mutual TLS verification.")
 	fs.StringVar(&s.ProxyClientKeyFile, "proxy-client-key-file", s.ProxyClientKeyFile, ""+
 		"Private key for the client certificate used to prove the identity of the aggregator or kube-apiserver "+
@@ -236,4 +235,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.EnableAggregatorRouting, "enable-aggregator-routing", s.EnableAggregatorRouting,
 		"Turns on aggregator routing requests to endoints IP rather than cluster IP.")
 
+	fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
+		"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key. (Ignored unless alpha TokenRequest is enabled")
+
 }

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/options/options_test.go

@@ -26,8 +26,9 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/diff"
 	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	genericoptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
-	utilconfig "k8s.io/apiserver/pkg/util/flag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
 	auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -43,7 +44,7 @@ func TestAddFlags(t *testing.T) {
 	s.AddFlags(f)
 
 	args := []string{
-		"--admission-control=AlwaysDeny",
+		"--enable-admission-plugins=AlwaysDeny",
 		"--admission-control-config-file=/admission-control-config",
 		"--advertise-address=192.168.10.10",
 		"--allow-privileged=false",
@@ -83,6 +84,7 @@ func TestAddFlags(t *testing.T) {
 		"--etcd-keyfile=/var/run/kubernetes/etcd.key",
 		"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
 		"--etcd-cafile=/var/run/kubernetes/etcdca.crt",
+		"--http2-max-streams-per-connection=42",
 		"--kubelet-https=true",
 		"--kubelet-read-only-port=10255",
 		"--kubelet-timeout=5s",
@@ -99,6 +101,7 @@ func TestAddFlags(t *testing.T) {
 	// This is a snapshot of expected options parsed by args.
 	expected := &ServerRunOptions{
 		ServiceNodePortRange:   kubeoptions.DefaultServiceNodePortRange,
+		ServiceClusterIPRange:  kubeoptions.DefaultServiceIPCIDR,
 		MasterCount:            5,
 		EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
 		AllowPrivileged:        false,
@@ -110,12 +113,14 @@ func TestAddFlags(t *testing.T) {
 			RequestTimeout:              time.Duration(2) * time.Minute,
 			MinRequestTimeout:           1800,
 		},
-		Admission: &apiserveroptions.AdmissionOptions{
-			RecommendedPluginOrder: []string{"NamespaceLifecycle", "Initializers", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook"},
-			DefaultOffPlugins:      []string{"Initializers", "MutatingAdmissionWebhook", "ValidatingAdmissionWebhook"},
-			PluginNames:            []string{"AlwaysDeny"},
-			ConfigFile:             "/admission-control-config",
-			Plugins:                s.Admission.Plugins,
+		Admission: &kubeoptions.AdmissionOptions{
+			GenericAdmission: &apiserveroptions.AdmissionOptions{
+				RecommendedPluginOrder: s.Admission.GenericAdmission.RecommendedPluginOrder,
+				DefaultOffPlugins:      s.Admission.GenericAdmission.DefaultOffPlugins,
+				EnablePlugins:          []string{"AlwaysDeny"},
+				ConfigFile:             "/admission-control-config",
+				Plugins:                s.Admission.GenericAdmission.Plugins,
+			},
 		},
 		Etcd: &apiserveroptions.EtcdOptions{
 			StorageConfig: storagebackend.Config{
@@ -123,11 +128,12 @@ func TestAddFlags(t *testing.T) {
 				ServerList: nil,
 				Prefix:     "/registry",
 				DeserializationCacheSize: 0,
-				Quorum:             false,
-				KeyFile:            "/var/run/kubernetes/etcd.key",
-				CAFile:             "/var/run/kubernetes/etcdca.crt",
-				CertFile:           "/var/run/kubernetes/etcdce.crt",
-				CompactionInterval: storagebackend.DefaultCompactInterval,
+				Quorum:                false,
+				KeyFile:               "/var/run/kubernetes/etcd.key",
+				CAFile:                "/var/run/kubernetes/etcdca.crt",
+				CertFile:              "/var/run/kubernetes/etcdce.crt",
+				CompactionInterval:    storagebackend.DefaultCompactInterval,
+				CountMetricPollPeriod: time.Minute,
 			},
 			DefaultStorageMediaType: "application/vnd.kubernetes.protobuf",
 			DeleteCollectionWorkers: 1,
@@ -135,14 +141,15 @@ func TestAddFlags(t *testing.T) {
 			EnableWatchCache:        true,
 			DefaultWatchCacheSize:   100,
 		},
-		SecureServing: &apiserveroptions.SecureServingOptions{
+		SecureServing: genericoptions.WithLoopback(&apiserveroptions.SecureServingOptions{
 			BindAddress: net.ParseIP("192.168.10.20"),
 			BindPort:    6443,
 			ServerCert: apiserveroptions.GeneratableKeyCert{
 				CertDirectory: "/var/run/kubernetes",
 				PairName:      "apiserver",
 			},
-		},
+			HTTP2MaxStreamsPerConnection: 42,
+		}),
 		InsecureServing: &kubeoptions.InsecureServingOptions{
 			BindAddress: net.ParseIP("127.0.0.1"),
 			BindPort:    8080,
@@ -205,9 +212,9 @@ func TestAddFlags(t *testing.T) {
 				ConfigFile: "/token-webhook-config",
 			},
 			BootstrapToken: &kubeoptions.BootstrapTokenAuthenticationOptions{},
-			Keystone:       &kubeoptions.KeystoneAuthenticationOptions{},
 			OIDC: &kubeoptions.OIDCAuthenticationOptions{
 				UsernameClaim: "sub",
+				SigningAlgs:   []string{"RS256"},
 			},
 			PasswordFile:  &kubeoptions.PasswordFileAuthenticationOptions{},
 			RequestHeader: &apiserveroptions.RequestHeaderAuthenticationOptions{},
@@ -233,8 +240,8 @@ func TestAddFlags(t *testing.T) {
 			StorageVersions:        legacyscheme.Registry.AllPreferredGroupVersions(),
 			DefaultStorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(),
 		},
-		APIEnablement: &kubeoptions.APIEnablementOptions{
-			RuntimeConfig: utilconfig.ConfigurationMap{},
+		APIEnablement: &apiserveroptions.APIEnablementOptions{
+			RuntimeConfig: utilflag.ConfigurationMap{},
 		},
 		EnableLogsHandler:       false,
 		EnableAggregatorRouting: true,

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/options/plugins.go

@@ -1,86 +0,0 @@
-/*
-Copyright 2014 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package options
-
-// This file exists to force the desired plugin implementations to be linked.
-// This should probably be part of some configuration fed into the build for a
-// given binary target.
-import (
-	// Cloud providers
-	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
-
-	// Admission policies
-	"k8s.io/apiserver/pkg/admission"
-	"k8s.io/kubernetes/plugin/pkg/admission/admit"
-	"k8s.io/kubernetes/plugin/pkg/admission/alwayspullimages"
-	"k8s.io/kubernetes/plugin/pkg/admission/antiaffinity"
-	"k8s.io/kubernetes/plugin/pkg/admission/defaulttolerationseconds"
-	"k8s.io/kubernetes/plugin/pkg/admission/deny"
-	"k8s.io/kubernetes/plugin/pkg/admission/eventratelimit"
-	"k8s.io/kubernetes/plugin/pkg/admission/exec"
-	"k8s.io/kubernetes/plugin/pkg/admission/extendedresourcetoleration"
-	"k8s.io/kubernetes/plugin/pkg/admission/gc"
-	"k8s.io/kubernetes/plugin/pkg/admission/imagepolicy"
-	"k8s.io/kubernetes/plugin/pkg/admission/initialresources"
-	"k8s.io/kubernetes/plugin/pkg/admission/limitranger"
-	"k8s.io/kubernetes/plugin/pkg/admission/namespace/autoprovision"
-	"k8s.io/kubernetes/plugin/pkg/admission/namespace/exists"
-	"k8s.io/kubernetes/plugin/pkg/admission/noderestriction"
-	"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/label"
-	"k8s.io/kubernetes/plugin/pkg/admission/persistentvolume/resize"
-	"k8s.io/kubernetes/plugin/pkg/admission/persistentvolumeclaim/pvcprotection"
-	"k8s.io/kubernetes/plugin/pkg/admission/podnodeselector"
-	"k8s.io/kubernetes/plugin/pkg/admission/podpreset"
-	"k8s.io/kubernetes/plugin/pkg/admission/podtolerationrestriction"
-	podpriority "k8s.io/kubernetes/plugin/pkg/admission/priority"
-	"k8s.io/kubernetes/plugin/pkg/admission/resourcequota"
-	"k8s.io/kubernetes/plugin/pkg/admission/security/podsecuritypolicy"
-	"k8s.io/kubernetes/plugin/pkg/admission/securitycontext/scdeny"
-	"k8s.io/kubernetes/plugin/pkg/admission/serviceaccount"
-	"k8s.io/kubernetes/plugin/pkg/admission/storageclass/setdefault"
-)
-
-// RegisterAllAdmissionPlugins registers all admission plugins
-func RegisterAllAdmissionPlugins(plugins *admission.Plugins) {
-	admit.Register(plugins)
-	alwayspullimages.Register(plugins)
-	antiaffinity.Register(plugins)
-	defaulttolerationseconds.Register(plugins)
-	deny.Register(plugins)
-	eventratelimit.Register(plugins)
-	exec.Register(plugins)
-	extendedresourcetoleration.Register(plugins)
-	gc.Register(plugins)
-	imagepolicy.Register(plugins)
-	initialresources.Register(plugins)
-	limitranger.Register(plugins)
-	autoprovision.Register(plugins)
-	exists.Register(plugins)
-	noderestriction.Register(plugins)
-	label.Register(plugins) // DEPRECATED in favor of NewPersistentVolumeLabelController in CCM
-	podnodeselector.Register(plugins)
-	podpreset.Register(plugins)
-	podtolerationrestriction.Register(plugins)
-	resourcequota.Register(plugins)
-	podsecuritypolicy.Register(plugins)
-	podpriority.Register(plugins)
-	scdeny.Register(plugins)
-	serviceaccount.Register(plugins)
-	setdefault.Register(plugins)
-	resize.Register(plugins)
-	pvcprotection.Register(plugins)
-}

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/options/validation.go

@@ -18,6 +18,10 @@ package options
 
 import (
 	"fmt"
+
+	apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
+	aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
 )
 
 // TODO: Longer term we should read this from some config store, rather than a flag.
@@ -66,11 +70,18 @@ func (options *ServerRunOptions) Validate() []error {
 	if errs := options.Audit.Validate(); len(errs) > 0 {
 		errors = append(errors, errs...)
 	}
+	if errs := options.Admission.Validate(); len(errs) > 0 {
+		errors = append(errors, errs...)
+	}
 	if errs := options.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
 		errors = append(errors, errs...)
 	}
 	if options.MasterCount <= 0 {
 		errors = append(errors, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", options.MasterCount))
 	}
+	if errs := options.APIEnablement.Validate(legacyscheme.Registry, apiextensionsapiserver.Registry, aggregatorscheme.Registry); len(errs) > 0 {
+		errors = append(errors, errs...)
+	}
+
 	return errors
 }

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/server.go

@@ -35,7 +35,6 @@ import (
 	"github.com/go-openapi/spec"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
 
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -45,21 +44,23 @@ import (
 	utilwait "k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/admission"
 	webhookconfig "k8s.io/apiserver/pkg/admission/plugin/webhook/config"
+	webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	"k8s.io/apiserver/pkg/server"
 	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/filters"
 	serveroptions "k8s.io/apiserver/pkg/server/options"
 	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
-	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
-	openapi "k8s.io/kube-openapi/pkg/common"
-
-	webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
 	"k8s.io/apiserver/pkg/storage/etcd3/preflight"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	clientgoinformers "k8s.io/client-go/informers"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	certutil "k8s.io/client-go/util/cert"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+	openapi "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
 	"k8s.io/kubernetes/pkg/apis/admissionregistration"
@@ -69,12 +70,14 @@ import (
 	"k8s.io/kubernetes/pkg/apis/events"
 	"k8s.io/kubernetes/pkg/apis/extensions"
 	"k8s.io/kubernetes/pkg/apis/networking"
+	"k8s.io/kubernetes/pkg/apis/policy"
 	"k8s.io/kubernetes/pkg/apis/storage"
 	"k8s.io/kubernetes/pkg/capabilities"
 	"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
 	informers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
+	"k8s.io/kubernetes/pkg/features"
 	generatedopenapi "k8s.io/kubernetes/pkg/generated/openapi"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
 	kubeapiserveradmission "k8s.io/kubernetes/pkg/kubeapiserver/admission"
@@ -88,9 +91,12 @@ import (
 	quotainstall "k8s.io/kubernetes/pkg/quota/install"
 	"k8s.io/kubernetes/pkg/registry/cachesize"
 	rbacrest "k8s.io/kubernetes/pkg/registry/rbac/rest"
+	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/version"
+	"k8s.io/kubernetes/pkg/version/verflag"
 	"k8s.io/kubernetes/plugin/pkg/auth/authenticator/token/bootstrap"
 
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
 	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
 )
@@ -101,7 +107,6 @@ const etcdRetryInterval = 1 * time.Second
 // NewAPIServerCommand creates a *cobra.Command object with default parameters
 func NewAPIServerCommand() *cobra.Command {
 	s := options.NewServerRunOptions()
-	s.AddFlags(pflag.CommandLine)
 	cmd := &cobra.Command{
 		Use: "kube-apiserver",
 		Long: `The Kubernetes API server validates and configures data
@@ -109,8 +114,17 @@ for the api objects which include pods, services, replicationcontrollers, and
 others. The API Server services REST operations and provides the frontend to the
 cluster's shared state through which all other components interact.`,
 		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
+			stopCh := server.SetupSignalHandler()
+			if err := Run(s, stopCh); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
 		},
 	}
+	s.AddFlags(cmd.Flags())
 
 	return cmd
 }
@@ -134,6 +148,7 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc
 	if err != nil {
 		return nil, err
 	}
+
 	kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
 	if err != nil {
 		return nil, err
@@ -296,7 +311,7 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		return nil, nil, nil, nil, nil, err
 	}
 
-	storageFactory, err := BuildStorageFactory(s)
+	storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -310,6 +325,28 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 		return nil, nil, nil, nil, nil, err
 	}
 
+	var issuer serviceaccount.TokenGenerator
+	var apiAudiences []string
+	if s.ServiceAccountSigningKeyFile != "" ||
+		s.Authentication.ServiceAccounts.Issuer != "" ||
+		len(s.Authentication.ServiceAccounts.APIAudiences) > 0 {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
+			return nil, nil, nil, nil, nil, fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file and/or --service-account-issuer-id flags were passed")
+		}
+		if s.ServiceAccountSigningKeyFile == "" ||
+			s.Authentication.ServiceAccounts.Issuer == "" ||
+			len(s.Authentication.ServiceAccounts.APIAudiences) == 0 ||
+			len(s.Authentication.ServiceAccounts.KeyFiles) == 0 {
+			return nil, nil, nil, nil, nil, fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together")
+		}
+		sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile)
+		if err != nil {
+			return nil, nil, nil, nil, nil, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
+		}
+		issuer = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk)
+		apiAudiences = s.Authentication.ServiceAccounts.APIAudiences
+	}
+
 	config := &master.Config{
 		GenericConfig: genericConfig,
 		ExtraConfig: master.ExtraConfig{
@@ -327,7 +364,6 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 			EnableCoreControllers:   true,
 			EventTTL:                s.EventTTL,
 			KubeletClientConfig:     s.KubeletConfig,
-			EnableUISupport:         true,
 			EnableLogsSupport:       s.EnableLogsHandler,
 			ProxyTransport:          proxyTransport,
 
@@ -342,6 +378,9 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
 
 			EndpointReconcilerType: reconcilers.Type(s.EndpointReconcilerType),
 			MasterCount:            s.MasterCount,
+
+			ServiceAccountIssuer:       issuer,
+			ServiceAccountAPIAudiences: apiAudiences,
 		},
 	}
 
@@ -376,6 +415,9 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 	if err := s.Features.ApplyTo(genericConfig); err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
+	if err := s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); err != nil {
+		return nil, nil, nil, nil, nil, err
+	}
 
 	genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, legacyscheme.Scheme)
 	genericConfig.OpenAPIConfig.PostProcessSpec = postProcessOpenAPISpecForBackwardCompatibility
@@ -390,7 +432,7 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 	kubeVersion := version.Get()
 	genericConfig.Version = &kubeVersion
 
-	storageFactory, err := BuildStorageFactory(s)
+	storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
 	if err != nil {
 		return nil, nil, nil, nil, nil, err
 	}
@@ -438,12 +480,12 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 		)
 	}
 
-	genericConfig.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, storageFactory, client, sharedInformers)
+	genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, storageFactory, client, clientgoExternalClient, sharedInformers)
 	if err != nil {
 		return nil, nil, nil, nil, nil, fmt.Errorf("invalid authentication config: %v", err)
 	}
 
-	genericConfig.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, sharedInformers)
+	genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, sharedInformers, versionedInformers)
 	if err != nil {
 		return nil, nil, nil, nil, nil, fmt.Errorf("invalid authorization config: %v", err)
 	}
@@ -451,27 +493,35 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
 		genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
 	}
 
-	webhookAuthResolver := func(delegate webhookconfig.AuthenticationInfoResolver) webhookconfig.AuthenticationInfoResolver {
-		return webhookconfig.AuthenticationInfoResolverFunc(func(server string) (*rest.Config, error) {
-			if server == "kubernetes.default.svc" {
-				return genericConfig.LoopbackClientConfig, nil
-			}
-			ret, err := delegate.ClientConfigFor(server)
-			if err != nil {
-				return nil, err
-			}
-			if proxyTransport != nil && proxyTransport.Dial != nil {
-				ret.Dial = proxyTransport.Dial
-			}
-			return ret, err
-		})
+	webhookAuthResolverWrapper := func(delegate webhookconfig.AuthenticationInfoResolver) webhookconfig.AuthenticationInfoResolver {
+		return &webhookconfig.AuthenticationInfoResolverDelegator{
+			ClientConfigForFunc: func(server string) (*rest.Config, error) {
+				if server == "kubernetes.default.svc" {
+					return genericConfig.LoopbackClientConfig, nil
+				}
+				return delegate.ClientConfigFor(server)
+			},
+			ClientConfigForServiceFunc: func(serviceName, serviceNamespace string) (*rest.Config, error) {
+				if serviceName == "kubernetes" && serviceNamespace == "default" {
+					return genericConfig.LoopbackClientConfig, nil
+				}
+				ret, err := delegate.ClientConfigForService(serviceName, serviceNamespace)
+				if err != nil {
+					return nil, err
+				}
+				if proxyTransport != nil && proxyTransport.Dial != nil {
+					ret.Dial = proxyTransport.Dial
+				}
+				return ret, err
+			},
+		}
 	}
 	pluginInitializers, err := BuildAdmissionPluginInitializers(
 		s,
 		client,
 		sharedInformers,
 		serviceResolver,
-		webhookAuthResolver,
+		webhookAuthResolverWrapper,
 	)
 	if err != nil {
 		return nil, nil, nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err)
@@ -514,25 +564,10 @@ func BuildAdmissionPluginInitializers(s *options.ServerRunOptions, client intern
 }
 
 // BuildAuthenticator constructs the authenticator
-func BuildAuthenticator(s *options.ServerRunOptions, storageFactory serverstorage.StorageFactory, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
+func BuildAuthenticator(s *options.ServerRunOptions, storageFactory serverstorage.StorageFactory, client internalclientset.Interface, extclient clientgoclientset.Interface, sharedInformers informers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
 	authenticatorConfig := s.Authentication.ToAuthenticationConfig()
 	if s.Authentication.ServiceAccounts.Lookup {
-		// we have to go direct to storage because the clientsets fail when they're initialized with some API versions excluded
-		// we should stop trying to control them like that.
-		storageConfigServiceAccounts, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("unable to get serviceaccounts storage: %v", err)
-		}
-		storageConfigSecrets, err := storageFactory.NewConfig(api.Resource("secrets"))
-		if err != nil {
-			return nil, nil, fmt.Errorf("unable to get secrets storage: %v", err)
-		}
-		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(
-			storageConfigServiceAccounts,
-			storageFactory.ResourcePrefix(api.Resource("serviceaccounts")),
-			storageConfigSecrets,
-			storageFactory.ResourcePrefix(api.Resource("secrets")),
-		)
+		authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(extclient)
 	}
 	if client == nil || reflect.ValueOf(client).IsNil() {
 		// TODO: Remove check once client can never be nil.
@@ -546,14 +581,14 @@ func BuildAuthenticator(s *options.ServerRunOptions, storageFactory serverstorag
 }
 
 // BuildAuthorizer constructs the authorizer
-func BuildAuthorizer(s *options.ServerRunOptions, sharedInformers informers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
-	authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
+func BuildAuthorizer(s *options.ServerRunOptions, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory) (authorizer.Authorizer, authorizer.RuleResolver, error) {
+	authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers, versionedInformers)
 	return authorizationConfig.New()
 }
 
 // BuildStorageFactory constructs the storage factory. If encryption at rest is used, it expects
 // all supported KMS plugins to be registered in the KMS plugin registry before being called.
-func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultStorageFactory, error) {
+func BuildStorageFactory(s *options.ServerRunOptions, apiResourceConfig *serverstorage.ResourceConfig) (*serverstorage.DefaultStorageFactory, error) {
 	storageGroupsToEncodingVersion, err := s.StorageSerialization.StorageGroupsToEncodingVersion()
 	if err != nil {
 		return nil, fmt.Errorf("error generating storage version map: %s", err)
@@ -566,21 +601,21 @@ func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultSto
 		// FIXME (soltysh): this GroupVersionResource override should be configurable
 		[]schema.GroupVersionResource{
 			batch.Resource("cronjobs").WithVersion("v1beta1"),
-			storage.Resource("volumeattachments").WithVersion("v1alpha1"),
+			storage.Resource("volumeattachments").WithVersion("v1beta1"),
 			admissionregistration.Resource("initializerconfigurations").WithVersion("v1alpha1"),
 		},
-		master.DefaultAPIResourceConfigSource(), s.APIEnablement.RuntimeConfig)
+		apiResourceConfig)
 	if err != nil {
 		return nil, fmt.Errorf("error in initializing storage factory: %s", err)
 	}
 
 	storageFactory.AddCohabitatingResources(networking.Resource("networkpolicies"), extensions.Resource("networkpolicies"))
-
-	// keep Deployments, Daemonsets and ReplicaSets in extensions for backwards compatibility, we'll have to migrate at some point, eventually
-	storageFactory.AddCohabitatingResources(extensions.Resource("deployments"), apps.Resource("deployments"))
-	storageFactory.AddCohabitatingResources(extensions.Resource("daemonsets"), apps.Resource("daemonsets"))
-	storageFactory.AddCohabitatingResources(extensions.Resource("replicasets"), apps.Resource("replicasets"))
+	storageFactory.AddCohabitatingResources(apps.Resource("deployments"), extensions.Resource("deployments"))
+	storageFactory.AddCohabitatingResources(apps.Resource("daemonsets"), extensions.Resource("daemonsets"))
+	storageFactory.AddCohabitatingResources(apps.Resource("replicasets"), extensions.Resource("replicasets"))
 	storageFactory.AddCohabitatingResources(api.Resource("events"), events.Resource("events"))
+	// TODO(#54933): 1.11: switch to using policy storage and flip the order here
+	storageFactory.AddCohabitatingResources(extensions.Resource("podsecuritypolicies"), policy.Resource("podsecuritypolicies"))
 	for _, override := range s.Etcd.EtcdServersOverrides {
 		tokens := strings.Split(override, "#")
 		if len(tokens) != 2 {
@@ -616,7 +651,7 @@ func BuildStorageFactory(s *options.ServerRunOptions) (*serverstorage.DefaultSto
 
 func defaultOptions(s *options.ServerRunOptions) error {
 	// set defaults
-	if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing); err != nil {
+	if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil {
 		return err
 	}
 	if err := kubeoptions.DefaultAdvertiseAddress(s.GenericServerRunOptions, s.InsecureServing); err != nil {
@@ -630,18 +665,35 @@ func defaultOptions(s *options.ServerRunOptions) error {
 	if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
 		return fmt.Errorf("error creating self-signed certificates: %v", err)
 	}
-	if err := s.CloudProvider.DefaultExternalHost(s.GenericServerRunOptions); err != nil {
-		return fmt.Errorf("error setting the external host value: %v", err)
+
+	if len(s.GenericServerRunOptions.ExternalHost) == 0 {
+		if len(s.GenericServerRunOptions.AdvertiseAddress) > 0 {
+			s.GenericServerRunOptions.ExternalHost = s.GenericServerRunOptions.AdvertiseAddress.String()
+		} else {
+			if hostname, err := os.Hostname(); err == nil {
+				s.GenericServerRunOptions.ExternalHost = hostname
+			} else {
+				return fmt.Errorf("error finding host name: %v", err)
+			}
+		}
+		glog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost)
 	}
 
 	s.Authentication.ApplyAuthorization(s.Authorization)
 
-	// Default to the private server key for service account token signing
-	if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
-		if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
-			s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
-		} else {
-			glog.Warning("No TLS key provided, service account token authentication disabled")
+	// Use (ServiceAccountSigningKeyFile != "") as a proxy to the user enabling
+	// TokenRequest functionality. This defaulting was convenient, but messed up
+	// a lot of people when they rotated their serving cert with no idea it was
+	// connected to their service account keys. We are taking this oppurtunity to
+	// remove this problematic defaulting.
+	if s.ServiceAccountSigningKeyFile == "" {
+		// Default to the private server key for service account token signing
+		if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
+			if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
+				s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
+			} else {
+				glog.Warning("No TLS key provided, service account token authentication disabled")
+			}
 		}
 	}
 
@@ -680,6 +732,20 @@ func defaultOptions(s *options.ServerRunOptions) error {
 		}
 	}
 
+	// TODO: remove when we stop supporting the legacy group version.
+	if s.APIEnablement.RuntimeConfig != nil {
+		for key, value := range s.APIEnablement.RuntimeConfig {
+			if key == "v1" || strings.HasPrefix(key, "v1/") ||
+				key == "api/v1" || strings.HasPrefix(key, "api/v1/") {
+				delete(s.APIEnablement.RuntimeConfig, key)
+				s.APIEnablement.RuntimeConfig["/v1"] = value
+			}
+			if key == "api/legacy" {
+				delete(s.APIEnablement.RuntimeConfig, key)
+			}
+		}
+	}
+
 	return nil
 }
 

vendor/k8s.io/kubernetes/cmd/kube-apiserver/app/testing/testserver.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
-	"testing"
 	"time"
 
 	pflag "github.com/spf13/pflag"
@@ -46,13 +45,20 @@ type TestServer struct {
 	TmpDir       string                    // Temp Dir used, by the apiserver
 }
 
+// Logger allows t.Testing and b.Testing to be passed to StartTestServer and StartTestServerOrDie
+type Logger interface {
+	Errorf(format string, args ...interface{})
+	Fatalf(format string, args ...interface{})
+	Logf(format string, args ...interface{})
+}
+
 // StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
 // and location of the tmpdir are returned.
 //
-// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
-// 		 files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
-// 		 enough time to remove temporariy files.
-func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
+// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
+// 		 files that because Golang testing's call to os.Exit will not give a stop channel go routine
+// 		 enough time to remove temporary files.
+func StartTestServer(t Logger, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
 
 	// TODO : Remove TrackStorageCleanup below when PR
 	// https://github.com/kubernetes/kubernetes/pull/50690
@@ -137,7 +143,7 @@ func StartTestServer(t *testing.T, customFlags []string, storageConfig *storageb
 }
 
 // StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
-func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer {
+func StartTestServerOrDie(t Logger, flags []string, storageConfig *storagebackend.Config) *TestServer {
 
 	result, err := StartTestServer(t, flags, storageConfig)
 	if err == nil {

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/BUILD

@@ -9,14 +9,8 @@ load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "kube-controller-manager",
-    gc_linkopts = [
-        "-linkmode",
-        "external",
-        "-extldflags",
-        "-static",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/kube-controller-manager",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
+    pure = "on",
     x_defs = version_x_defs(),
 )
 
@@ -26,12 +20,10 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager",
     deps = [
         "//cmd/kube-controller-manager/app:go_default_library",
-        "//cmd/kube-controller-manager/app/options:go_default_library",
         "//pkg/client/metrics/prometheus:go_default_library",
         "//pkg/util/reflector/prometheus:go_default_library",
         "//pkg/util/workqueue/prometheus:go_default_library",
         "//pkg/version/prometheus:go_default_library",
-        "//pkg/version/verflag:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/BUILD

@@ -1,10 +1,4 @@
-package(default_visibility = ["//visibility:public"])
-
-load(
-    "@io_bazel_rules_go//go:def.bzl",
-    "go_library",
-    "go_test",
-)
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "go_default_library",
@@ -14,6 +8,7 @@ go_library(
         "batch.go",
         "bootstrap.go",
         "certificates.go",
+        "cloudproviders.go",
         "controllermanager.go",
         "core.go",
         "extensions.go",
@@ -23,7 +18,11 @@ go_library(
         "rbac.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app",
+    visibility = ["//visibility:public"],
     deps = [
+        "//cmd/controller-manager/app:go_default_library",
+        "//cmd/controller-manager/app/options:go_default_library",
+        "//cmd/kube-controller-manager/app/config:go_default_library",
         "//cmd/kube-controller-manager/app/options:go_default_library",
         "//pkg/api/legacyscheme:go_default_library",
         "//pkg/apis/apps/install:go_default_library",
@@ -57,8 +56,9 @@ go_library(
         "//pkg/controller/garbagecollector:go_default_library",
         "//pkg/controller/job:go_default_library",
         "//pkg/controller/namespace:go_default_library",
-        "//pkg/controller/node:go_default_library",
-        "//pkg/controller/node/ipam:go_default_library",
+        "//pkg/controller/nodeipam:go_default_library",
+        "//pkg/controller/nodeipam/ipam:go_default_library",
+        "//pkg/controller/nodelifecycle:go_default_library",
         "//pkg/controller/podautoscaler:go_default_library",
         "//pkg/controller/podautoscaler/metrics:go_default_library",
         "//pkg/controller/podgc:go_default_library",
@@ -74,13 +74,16 @@ go_library(
         "//pkg/controller/volume/expand:go_default_library",
         "//pkg/controller/volume/persistentvolume:go_default_library",
         "//pkg/controller/volume/pvcprotection:go_default_library",
+        "//pkg/controller/volume/pvprotection:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/quota/generic:go_default_library",
         "//pkg/quota/install:go_default_library",
         "//pkg/serviceaccount:go_default_library",
         "//pkg/util/configz:go_default_library",
+        "//pkg/util/flag:go_default_library",
         "//pkg/util/metrics:go_default_library",
         "//pkg/version:go_default_library",
+        "//pkg/version/verflag:go_default_library",
         "//pkg/volume:go_default_library",
         "//pkg/volume/aws_ebs:go_default_library",
         "//pkg/volume/azure_dd:go_default_library",
@@ -105,32 +108,38 @@ go_library(
         "//pkg/volume/util:go_default_library",
         "//pkg/volume/vsphere_volume:go_default_library",
         "//vendor/github.com/golang/glog:go_default_library",
-        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
         "//vendor/github.com/spf13/cobra:go_default_library",
-        "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/meta:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
-        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/discovery/cached:go_default_library",
         "//vendor/k8s.io/client-go/dynamic:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
-        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
         "//vendor/k8s.io/client-go/scale:go_default_library",
-        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
-        "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/client-go/util/cert:go_default_library",
         "//vendor/k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1:go_default_library",
         "//vendor/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
+        "//vendor/k8s.io/metrics/pkg/client/external_metrics:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["controller_manager_test.go"],
+    embed = [":go_default_library"],
+    deps = [
+        "//vendor/github.com/stretchr/testify/assert:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
     ],
 )
 
@@ -145,18 +154,9 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//cmd/kube-controller-manager/app/config:all-srcs",
         "//cmd/kube-controller-manager/app/options:all-srcs",
     ],
     tags = ["automanaged"],
-)
-
-go_test(
-    name = "go_default_test",
-    srcs = ["controller_manager_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app",
-    library = ":go_default_library",
-    deps = [
-        "//vendor/github.com/stretchr/testify/assert:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
-    ],
+    visibility = ["//visibility:public"],
 )

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/apps.go

@@ -21,19 +21,40 @@ limitations under the License.
 package app
 
 import (
+	"fmt"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/kubernetes/pkg/controller/daemon"
 	"k8s.io/kubernetes/pkg/controller/statefulset"
 )
 
+func startDaemonSetController(ctx ControllerContext) (bool, error) {
+	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"}] {
+		return false, nil
+	}
+	dsc, err := daemon.NewDaemonSetsController(
+		ctx.InformerFactory.Apps().V1().DaemonSets(),
+		ctx.InformerFactory.Apps().V1().ControllerRevisions(),
+		ctx.InformerFactory.Core().V1().Pods(),
+		ctx.InformerFactory.Core().V1().Nodes(),
+		ctx.ClientBuilder.ClientOrDie("daemon-set-controller"),
+	)
+	if err != nil {
+		return true, fmt.Errorf("error creating DaemonSets controller: %v", err)
+	}
+	go dsc.Run(int(ctx.ComponentConfig.ConcurrentDaemonSetSyncs), ctx.Stop)
+	return true, nil
+}
+
 func startStatefulSetController(ctx ControllerContext) (bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "apps", Version: "v1beta1", Resource: "statefulsets"}] {
 		return false, nil
 	}
 	go statefulset.NewStatefulSetController(
 		ctx.InformerFactory.Core().V1().Pods(),
-		ctx.InformerFactory.Apps().V1beta1().StatefulSets(),
+		ctx.InformerFactory.Apps().V1().StatefulSets(),
 		ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
-		ctx.InformerFactory.Apps().V1beta1().ControllerRevisions(),
+		ctx.InformerFactory.Apps().V1().ControllerRevisions(),
 		ctx.ClientBuilder.ClientOrDie("statefulset-controller"),
 	).Run(1, ctx.Stop)
 	return true, nil

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/autoscaling.go

@@ -24,13 +24,14 @@ import (
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/discovery"
-	discocache "k8s.io/client-go/discovery/cached" // Saturday Night Fever
+	discocache "k8s.io/client-go/discovery/cached"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/scale"
 	"k8s.io/kubernetes/pkg/controller/podautoscaler"
 	"k8s.io/kubernetes/pkg/controller/podautoscaler/metrics"
 	resourceclient "k8s.io/metrics/pkg/client/clientset_generated/clientset/typed/metrics/v1beta1"
 	"k8s.io/metrics/pkg/client/custom_metrics"
+	"k8s.io/metrics/pkg/client/external_metrics"
 )
 
 func startHPAController(ctx ControllerContext) (bool, error) {
@@ -38,7 +39,7 @@ func startHPAController(ctx ControllerContext) (bool, error) {
 		return false, nil
 	}
 
-	if ctx.Options.HorizontalPodAutoscalerUseRESTClients {
+	if ctx.ComponentConfig.HorizontalPodAutoscalerUseRESTClients {
 		// use the new-style clients if support for custom metrics is enabled
 		return startHPAControllerWithRESTClient(ctx)
 	}
@@ -51,6 +52,7 @@ func startHPAControllerWithRESTClient(ctx ControllerContext) (bool, error) {
 	metricsClient := metrics.NewRESTMetricsClient(
 		resourceclient.NewForConfigOrDie(clientConfig),
 		custom_metrics.NewForConfigOrDie(clientConfig),
+		external_metrics.NewForConfigOrDie(clientConfig),
 	)
 	return startHPAControllerWithMetricsClient(ctx, metricsClient)
 }
@@ -88,7 +90,7 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 	replicaCalc := podautoscaler.NewReplicaCalculator(
 		metricsClient,
 		hpaClient.CoreV1(),
-		ctx.Options.HorizontalPodAutoscalerTolerance,
+		ctx.ComponentConfig.HorizontalPodAutoscalerTolerance,
 	)
 	go podautoscaler.NewHorizontalController(
 		hpaClientGoClient.CoreV1(),
@@ -97,9 +99,9 @@ func startHPAControllerWithMetricsClient(ctx ControllerContext, metricsClient me
 		restMapper,
 		replicaCalc,
 		ctx.InformerFactory.Autoscaling().V1().HorizontalPodAutoscalers(),
-		ctx.Options.HorizontalPodAutoscalerSyncPeriod.Duration,
-		ctx.Options.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
-		ctx.Options.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration,
+		ctx.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration,
 	).Run(ctx.Stop)
 	return true, nil
 }

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/batch.go

@@ -36,7 +36,7 @@ func startJobController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.InformerFactory.Batch().V1().Jobs(),
 		ctx.ClientBuilder.ClientOrDie("job-controller"),
-	).Run(int(ctx.Options.ConcurrentJobSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentJobSyncs), ctx.Stop)
 	return true, nil
 }
 

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/bootstrap.go

@@ -25,6 +25,8 @@ import (
 func startBootstrapSignerController(ctx ControllerContext) (bool, error) {
 	bsc, err := bootstrap.NewBootstrapSigner(
 		ctx.ClientBuilder.ClientGoClientOrDie("bootstrap-signer"),
+		ctx.InformerFactory.Core().V1().Secrets(),
+		ctx.InformerFactory.Core().V1().ConfigMaps(),
 		bootstrap.DefaultBootstrapSignerOptions(),
 	)
 	if err != nil {
@@ -37,6 +39,7 @@ func startBootstrapSignerController(ctx ControllerContext) (bool, error) {
 func startTokenCleanerController(ctx ControllerContext) (bool, error) {
 	tcc, err := bootstrap.NewTokenCleaner(
 		ctx.ClientBuilder.ClientGoClientOrDie("token-cleaner"),
+		ctx.InformerFactory.Core().V1().Secrets(),
 		bootstrap.DefaultTokenCleanerOptions(),
 	)
 	if err != nil {

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/certificates.go

@@ -27,7 +27,7 @@ import (
 	"github.com/golang/glog"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/controller/certificates/approver"
 	"k8s.io/kubernetes/pkg/controller/certificates/cleaner"
 	"k8s.io/kubernetes/pkg/controller/certificates/signer"
@@ -37,7 +37,7 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "certificates.k8s.io", Version: "v1beta1", Resource: "certificatesigningrequests"}] {
 		return false, nil
 	}
-	if ctx.Options.ClusterSigningCertFile == "" || ctx.Options.ClusterSigningKeyFile == "" {
+	if ctx.ComponentConfig.ClusterSigningCertFile == "" || ctx.ComponentConfig.ClusterSigningKeyFile == "" {
 		return false, nil
 	}
 
@@ -52,15 +52,15 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	// bail out of startController without logging.
 	var keyFileExists, keyUsesDefault, certFileExists, certUsesDefault bool
 
-	_, err := os.Stat(ctx.Options.ClusterSigningCertFile)
+	_, err := os.Stat(ctx.ComponentConfig.ClusterSigningCertFile)
 	certFileExists = !os.IsNotExist(err)
 
-	certUsesDefault = (ctx.Options.ClusterSigningCertFile == options.DefaultClusterSigningCertFile)
+	certUsesDefault = (ctx.ComponentConfig.ClusterSigningCertFile == cmoptions.DefaultClusterSigningCertFile)
 
-	_, err = os.Stat(ctx.Options.ClusterSigningKeyFile)
+	_, err = os.Stat(ctx.ComponentConfig.ClusterSigningKeyFile)
 	keyFileExists = !os.IsNotExist(err)
 
-	keyUsesDefault = (ctx.Options.ClusterSigningKeyFile == options.DefaultClusterSigningKeyFile)
+	keyUsesDefault = (ctx.ComponentConfig.ClusterSigningKeyFile == cmoptions.DefaultClusterSigningKeyFile)
 
 	switch {
 	case (keyFileExists && keyUsesDefault) || (certFileExists && certUsesDefault):
@@ -84,9 +84,9 @@ func startCSRSigningController(ctx ControllerContext) (bool, error) {
 	signer, err := signer.NewCSRSigningController(
 		c,
 		ctx.InformerFactory.Certificates().V1beta1().CertificateSigningRequests(),
-		ctx.Options.ClusterSigningCertFile,
-		ctx.Options.ClusterSigningKeyFile,
-		ctx.Options.ClusterSigningDuration.Duration,
+		ctx.ComponentConfig.ClusterSigningCertFile,
+		ctx.ComponentConfig.ClusterSigningKeyFile,
+		ctx.ComponentConfig.ClusterSigningDuration.Duration,
 	)
 	if err != nil {
 		return false, fmt.Errorf("failed to start certificate controller: %v", err)

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/cloudproviders.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+
+	"github.com/golang/glog"
+
+	"k8s.io/client-go/informers"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+// createCloudProvider helps consolidate what is needed for cloud providers, we explicitly list the things
+// that the cloud providers need as parameters, so we can control
+func createCloudProvider(cloudProvider string, externalCloudVolumePlugin string, cloudConfigFile string,
+	allowUntaggedCloud bool, sharedInformers informers.SharedInformerFactory) (cloudprovider.Interface, ControllerLoopMode, error) {
+	var cloud cloudprovider.Interface
+	var loopMode ControllerLoopMode
+	var err error
+	if cloudprovider.IsExternal(cloudProvider) {
+		loopMode = ExternalLoops
+		if externalCloudVolumePlugin == "" {
+			// externalCloudVolumePlugin is temporary until we split all cloud providers out.
+			// So we just tell the caller that we need to run ExternalLoops without any cloud provider.
+			return nil, loopMode, nil
+		}
+		cloud, err = cloudprovider.InitCloudProvider(externalCloudVolumePlugin, cloudConfigFile)
+	} else {
+		loopMode = IncludeCloudLoops
+		cloud, err = cloudprovider.InitCloudProvider(cloudProvider, cloudConfigFile)
+	}
+	if err != nil {
+		return nil, loopMode, fmt.Errorf("cloud provider could not be initialized: %v", err)
+	}
+
+	if cloud != nil && cloud.HasClusterID() == false {
+		if allowUntaggedCloud == true {
+			glog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
+		} else {
+			return nil, loopMode, fmt.Errorf("no ClusterID Found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
+		}
+	}
+
+	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
+		informerUserCloud.SetInformers(sharedInformers)
+	}
+	return cloud, loopMode, err
+}

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/config/BUILD

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/config",
+    visibility = ["//visibility:public"],
+    deps = ["//cmd/controller-manager/app:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/config/config.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"time"
+
+	genericcontrollermanager "k8s.io/kubernetes/cmd/controller-manager/app"
+)
+
+// ExtraConfig are part of Config, also can place your custom config here.
+type ExtraConfig struct {
+	NodeStatusUpdateFrequency time.Duration
+}
+
+// Config is the main context object for the controller manager.
+type Config struct {
+	Generic genericcontrollermanager.Config
+	Extra   ExtraConfig
+}
+
+type completedConfig struct {
+	Generic genericcontrollermanager.CompletedConfig
+	Extra   *ExtraConfig
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{
+		c.Generic.Complete(),
+		&c.Extra,
+	}
+
+	return &CompletedConfig{&cc}
+}

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/controllermanager.go

@@ -24,46 +24,34 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
-	"net"
-	"net/http"
-	"net/http/pprof"
 	"os"
-	goruntime "runtime"
-	"strconv"
 	"time"
 
+	"github.com/golang/glog"
+	"github.com/spf13/cobra"
+
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
-
-	"k8s.io/apiserver/pkg/server/healthz"
-
-	"k8s.io/api/core/v1"
-	"k8s.io/client-go/discovery"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	restclient "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/client-go/tools/record"
-	certutil "k8s.io/client-go/util/cert"
-
 	"k8s.io/client-go/informers"
-	clientset "k8s.io/client-go/kubernetes"
+	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	certutil "k8s.io/client-go/util/cert"
+	genericcontrollerconfig "k8s.io/kubernetes/cmd/controller-manager/app"
+	"k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
-	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/cloudprovider"
 	"k8s.io/kubernetes/pkg/controller"
 	serviceaccountcontroller "k8s.io/kubernetes/pkg/controller/serviceaccount"
 	"k8s.io/kubernetes/pkg/serviceaccount"
 	"k8s.io/kubernetes/pkg/util/configz"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	"k8s.io/kubernetes/pkg/version"
-
-	"github.com/golang/glog"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/spf13/cobra"
-	"github.com/spf13/pflag"
+	"k8s.io/kubernetes/pkg/version/verflag"
 )
 
 const (
@@ -71,10 +59,16 @@ const (
 	ControllerStartJitter = 1.0
 )
 
+type ControllerLoopMode int
+
+const (
+	IncludeCloudLoops ControllerLoopMode = iota
+	ExternalLoops
+)
+
 // NewControllerManagerCommand creates a *cobra.Command object with default parameters
 func NewControllerManagerCommand() *cobra.Command {
-	s := options.NewCMServer()
-	s.AddFlags(pflag.CommandLine, KnownControllers(), ControllersDisabledByDefault.List())
+	s := options.NewKubeControllerManagerOptions()
 	cmd := &cobra.Command{
 		Use: "kube-controller-manager",
 		Long: `The Kubernetes controller manager is a daemon that embeds
@@ -86,8 +80,22 @@ current state towards the desired state. Examples of controllers that ship with
 Kubernetes today are the replication controller, endpoints controller, namespace
 controller, and serviceaccounts controller.`,
 		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
+			c, err := s.Config(KnownControllers(), ControllersDisabledByDefault.List())
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
+
+			if err := Run(c.Complete()); err != nil {
+				fmt.Fprintf(os.Stderr, "%v\n", err)
+				os.Exit(1)
+			}
 		},
 	}
+	s.AddFlags(cmd.Flags(), KnownControllers(), ControllersDisabledByDefault.List())
 
 	return cmd
 }
@@ -95,63 +103,64 @@ controller, and serviceaccounts controller.`,
 // ResyncPeriod returns a function which generates a duration each time it is
 // invoked; this is so that multiple controllers don't get into lock-step and all
 // hammer the apiserver with list requests simultaneously.
-func ResyncPeriod(s *options.CMServer) func() time.Duration {
+func ResyncPeriod(c *config.CompletedConfig) func() time.Duration {
 	return func() time.Duration {
 		factor := rand.Float64() + 1
-		return time.Duration(float64(s.MinResyncPeriod.Nanoseconds()) * factor)
+		return time.Duration(float64(c.Generic.ComponentConfig.MinResyncPeriod.Nanoseconds()) * factor)
 	}
 }
 
-// Run runs the CMServer.  This should never exit.
-func Run(s *options.CMServer) error {
+// Run runs the KubeControllerManagerOptions.  This should never exit.
+func Run(c *config.CompletedConfig) error {
 	// To help debugging, immediately log version
 	glog.Infof("Version: %+v", version.Get())
-	if err := s.Validate(KnownControllers(), ControllersDisabledByDefault.List()); err != nil {
-		return err
-	}
 
-	if c, err := configz.New("componentconfig"); err == nil {
-		c.Set(s.KubeControllerManagerConfiguration)
+	if cfgz, err := configz.New("componentconfig"); err == nil {
+		cfgz.Set(c.Generic.ComponentConfig)
 	} else {
-		glog.Errorf("unable to register configz: %s", err)
+		glog.Errorf("unable to register configz: %c", err)
 	}
 
-	kubeClient, leaderElectionClient, kubeconfig, err := createClients(s)
-	if err != nil {
-		return err
+	// Start the controller manager HTTP server
+	stopCh := make(chan struct{})
+	if c.Generic.SecureServing != nil {
+		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh); err != nil {
+			return err
+		}
+	}
+	if c.Generic.InsecureServing != nil {
+		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
+			return err
+		}
 	}
-
-	go startHTTP(s)
-
-	recorder := createRecorder(kubeClient)
 
 	run := func(stop <-chan struct{}) {
 		rootClientBuilder := controller.SimpleControllerClientBuilder{
-			ClientConfig: kubeconfig,
+			ClientConfig: c.Generic.Kubeconfig,
 		}
 		var clientBuilder controller.ControllerClientBuilder
-		if s.UseServiceAccountCredentials {
-			if len(s.ServiceAccountKeyFile) == 0 {
-				// It's possible another controller process is creating the tokens for us.
+		if c.Generic.ComponentConfig.UseServiceAccountCredentials {
+			if len(c.Generic.ComponentConfig.ServiceAccountKeyFile) == 0 {
+				// It'c possible another controller process is creating the tokens for us.
 				// If one isn't, we'll timeout and exit when our client builder is unable to create the tokens.
 				glog.Warningf("--use-service-account-credentials was specified without providing a --service-account-private-key-file")
 			}
 			clientBuilder = controller.SAControllerClientBuilder{
-				ClientConfig:         restclient.AnonymousClientConfig(kubeconfig),
-				CoreClient:           kubeClient.CoreV1(),
-				AuthenticationClient: kubeClient.Authentication(),
+				ClientConfig:         restclient.AnonymousClientConfig(c.Generic.Kubeconfig),
+				CoreClient:           c.Generic.Client.CoreV1(),
+				AuthenticationClient: c.Generic.Client.AuthenticationV1(),
 				Namespace:            "kube-system",
 			}
 		} else {
 			clientBuilder = rootClientBuilder
 		}
-		ctx, err := CreateControllerContext(s, rootClientBuilder, clientBuilder, stop)
+		ctx, err := CreateControllerContext(c, rootClientBuilder, clientBuilder, stop)
 		if err != nil {
 			glog.Fatalf("error building controller context: %v", err)
 		}
 		saTokenControllerInitFunc := serviceAccountTokenControllerStarter{rootClientBuilder: rootClientBuilder}.startServiceAccountTokenController
 
-		if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers()); err != nil {
+		if err := StartControllers(ctx, saTokenControllerInitFunc, NewControllerInitializers(ctx.LoopMode)); err != nil {
 			glog.Fatalf("error starting controllers: %v", err)
 		}
 
@@ -161,8 +170,8 @@ func Run(s *options.CMServer) error {
 		select {}
 	}
 
-	if !s.LeaderElection.LeaderElect {
-		run(nil)
+	if !c.Generic.ComponentConfig.LeaderElection.LeaderElect {
+		run(wait.NeverStop)
 		panic("unreachable")
 	}
 
@@ -171,13 +180,15 @@ func Run(s *options.CMServer) error {
 		return err
 	}
 
-	rl, err := resourcelock.New(s.LeaderElection.ResourceLock,
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id = id + "_" + string(uuid.NewUUID())
+	rl, err := resourcelock.New(c.Generic.ComponentConfig.LeaderElection.ResourceLock,
 		"kube-system",
 		"kube-controller-manager",
-		leaderElectionClient.CoreV1(),
+		c.Generic.LeaderElectionClient.CoreV1(),
 		resourcelock.ResourceLockConfig{
 			Identity:      id,
-			EventRecorder: recorder,
+			EventRecorder: c.Generic.EventRecorder,
 		})
 	if err != nil {
 		glog.Fatalf("error creating lock: %v", err)
@@ -185,9 +196,9 @@ func Run(s *options.CMServer) error {
 
 	leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
 		Lock:          rl,
-		LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
-		RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
-		RetryPeriod:   s.LeaderElection.RetryPeriod.Duration,
+		LeaseDuration: c.Generic.ComponentConfig.LeaderElection.LeaseDuration.Duration,
+		RenewDeadline: c.Generic.ComponentConfig.LeaderElection.RenewDeadline.Duration,
+		RetryPeriod:   c.Generic.ComponentConfig.LeaderElection.RetryPeriod.Duration,
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: run,
 			OnStoppedLeading: func() {
@@ -198,53 +209,6 @@ func Run(s *options.CMServer) error {
 	panic("unreachable")
 }
 
-func startHTTP(s *options.CMServer) {
-	mux := http.NewServeMux()
-	healthz.InstallHandler(mux)
-	if s.EnableProfiling {
-		mux.HandleFunc("/debug/pprof/", pprof.Index)
-		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
-		if s.EnableContentionProfiling {
-			goruntime.SetBlockProfileRate(1)
-		}
-	}
-	configz.InstallHandler(mux)
-	mux.Handle("/metrics", prometheus.Handler())
-
-	server := &http.Server{
-		Addr:    net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))),
-		Handler: mux,
-	}
-	glog.Fatal(server.ListenAndServe())
-}
-
-func createRecorder(kubeClient *clientset.Clientset) record.EventRecorder {
-	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartLogging(glog.Infof)
-	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: v1core.New(kubeClient.CoreV1().RESTClient()).Events("")})
-	return eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: "controller-manager"})
-}
-
-func createClients(s *options.CMServer) (*clientset.Clientset, *clientset.Clientset, *restclient.Config, error) {
-	kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
-	if err != nil {
-		return nil, nil, nil, err
-	}
-
-	kubeconfig.ContentConfig.ContentType = s.ContentType
-	// Override kubeconfig qps/burst settings from flags
-	kubeconfig.QPS = s.KubeAPIQPS
-	kubeconfig.Burst = int(s.KubeAPIBurst)
-	kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager"))
-	if err != nil {
-		glog.Fatalf("Invalid API configuration: %v", err)
-	}
-	leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
-	return kubeClient, leaderElectionClient, kubeconfig, nil
-}
-
 type ControllerContext struct {
 	// ClientBuilder will provide a client for this controller to use
 	ClientBuilder controller.ControllerClientBuilder
@@ -253,7 +217,7 @@ type ControllerContext struct {
 	InformerFactory informers.SharedInformerFactory
 
 	// Options provides access to init options for a given controller
-	Options options.CMServer
+	ComponentConfig componentconfig.KubeControllerManagerConfiguration
 
 	// AvailableResources is a map listing currently available resources
 	AvailableResources map[schema.GroupVersionResource]bool
@@ -262,16 +226,26 @@ type ControllerContext struct {
 	// It must be initialized and ready to use.
 	Cloud cloudprovider.Interface
 
+	// Control for which control loops to be run
+	// IncludeCloudLoops is for a kube-controller-manager running all loops
+	// ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager
+	LoopMode ControllerLoopMode
+
 	// Stop is the stop channel
 	Stop <-chan struct{}
 
 	// InformersStarted is closed after all of the controllers have been initialized and are running.  After this point it is safe,
 	// for an individual controller to start the shared informers. Before it is closed, they should not.
 	InformersStarted chan struct{}
+
+	// ResyncPeriod generates a duration each time it is invoked; this is so that
+	// multiple controllers don't get into lock-step and all hammer the apiserver
+	// with list requests simultaneously.
+	ResyncPeriod func() time.Duration
 }
 
 func (c ControllerContext) IsControllerEnabled(name string) bool {
-	return IsControllerEnabled(name, ControllersDisabledByDefault, c.Options.Controllers...)
+	return IsControllerEnabled(name, ControllersDisabledByDefault, c.ComponentConfig.Controllers...)
 }
 
 func IsControllerEnabled(name string, disabledByDefaultControllers sets.String, controllers ...string) bool {
@@ -305,7 +279,7 @@ func IsControllerEnabled(name string, disabledByDefaultControllers sets.String,
 type InitFunc func(ctx ControllerContext) (bool, error)
 
 func KnownControllers() []string {
-	ret := sets.StringKeySet(NewControllerInitializers())
+	ret := sets.StringKeySet(NewControllerInitializers(IncludeCloudLoops))
 
 	// add "special" controllers that aren't initialized normally.  These controllers cannot be initialized
 	// using a normal function.  The only known special case is the SA token controller which *must* be started
@@ -329,7 +303,7 @@ const (
 
 // NewControllerInitializers is a public map of named controller groups (you can start more than one in an init func)
 // paired to their InitFunc.  This allows for structured downstream composition and subdivision.
-func NewControllerInitializers() map[string]InitFunc {
+func NewControllerInitializers(loopMode ControllerLoopMode) map[string]InitFunc {
 	controllers := map[string]InitFunc{}
 	controllers["endpoint"] = startEndpointController
 	controllers["replicationcontroller"] = startReplicationController
@@ -352,14 +326,20 @@ func NewControllerInitializers() map[string]InitFunc {
 	controllers["ttl"] = startTTLController
 	controllers["bootstrapsigner"] = startBootstrapSignerController
 	controllers["tokencleaner"] = startTokenCleanerController
-	controllers["service"] = startServiceController
-	controllers["node"] = startNodeController
-	controllers["route"] = startRouteController
+	if loopMode == IncludeCloudLoops {
+		controllers["service"] = startServiceController
+		controllers["nodeipam"] = startNodeIpamController
+		controllers["route"] = startRouteController
+		// TODO: volume controller into the IncludeCloudLoops only set.
+		// TODO: Separate cluster in cloud check from node lifecycle controller.
+	}
+	controllers["nodelifecycle"] = startNodeLifecycleController
 	controllers["persistentvolume-binder"] = startPersistentVolumeBinderController
 	controllers["attachdetach"] = startAttachDetachController
 	controllers["persistentvolume-expander"] = startVolumeExpandController
 	controllers["clusterrole-aggregation"] = startClusterRoleAggregrationController
 	controllers["pvc-protection"] = startPVCProtectionController
+	controllers["pv-protection"] = startPVProtectionController
 
 	return controllers
 }
@@ -368,34 +348,8 @@ func NewControllerInitializers() map[string]InitFunc {
 //  users don't have to restart their controller manager if they change the apiserver.
 // Until we get there, the structure here needs to be exposed for the construction of a proper ControllerContext.
 func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (map[schema.GroupVersionResource]bool, error) {
-	var discoveryClient discovery.DiscoveryInterface
-
-	var healthzContent string
-	// If apiserver is not running we should wait for some time and fail only then. This is particularly
-	// important when we start apiserver and controller manager at the same time.
-	err := wait.PollImmediate(time.Second, 10*time.Second, func() (bool, error) {
-		client, err := clientBuilder.Client("controller-discovery")
-		if err != nil {
-			glog.Errorf("Failed to get api versions from server: %v", err)
-			return false, nil
-		}
-
-		healthStatus := 0
-		resp := client.Discovery().RESTClient().Get().AbsPath("/healthz").Do().StatusCode(&healthStatus)
-		if healthStatus != http.StatusOK {
-			glog.Errorf("Server isn't healthy yet.  Waiting a little while.")
-			return false, nil
-		}
-		content, _ := resp.Raw()
-		healthzContent = string(content)
-
-		discoveryClient = client.Discovery()
-		return true, nil
-	})
-	if err != nil {
-		return nil, fmt.Errorf("failed to get api versions from server: %v: %v", healthzContent, err)
-	}
-
+	client := clientBuilder.ClientOrDie("controller-discovery")
+	discoveryClient := client.Discovery()
 	resourceMap, err := discoveryClient.ServerResources()
 	if err != nil {
 		utilruntime.HandleError(fmt.Errorf("unable to get all supported resources from server: %v", err))
@@ -421,40 +375,37 @@ func GetAvailableResources(clientBuilder controller.ControllerClientBuilder) (ma
 // CreateControllerContext creates a context struct containing references to resources needed by the
 // controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
 // the shared-informers client and token controller.
-func CreateControllerContext(s *options.CMServer, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
+func CreateControllerContext(s *config.CompletedConfig, rootClientBuilder, clientBuilder controller.ControllerClientBuilder, stop <-chan struct{}) (ControllerContext, error) {
 	versionedClient := rootClientBuilder.ClientOrDie("shared-informers")
 	sharedInformers := informers.NewSharedInformerFactory(versionedClient, ResyncPeriod(s)())
 
+	// If apiserver is not running we should wait for some time and fail only then. This is particularly
+	// important when we start apiserver and controller manager at the same time.
+	if err := genericcontrollerconfig.WaitForAPIServer(versionedClient, 10*time.Second); err != nil {
+		return ControllerContext{}, fmt.Errorf("failed to wait for apiserver being healthy: %v", err)
+	}
+
 	availableResources, err := GetAvailableResources(rootClientBuilder)
 	if err != nil {
 		return ControllerContext{}, err
 	}
 
-	cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
+	cloud, loopMode, err := createCloudProvider(s.Generic.ComponentConfig.CloudProvider, s.Generic.ComponentConfig.ExternalCloudVolumePlugin,
+		s.Generic.ComponentConfig.CloudConfigFile, s.Generic.ComponentConfig.AllowUntaggedCloud, sharedInformers)
 	if err != nil {
-		return ControllerContext{}, fmt.Errorf("cloud provider could not be initialized: %v", err)
-	}
-
-	if cloud != nil && cloud.HasClusterID() == false {
-		if s.AllowUntaggedCloud == true {
-			glog.Warning("detected a cluster without a ClusterID.  A ClusterID will be required in the future.  Please tag your cluster to avoid any future issues")
-		} else {
-			return ControllerContext{}, fmt.Errorf("no ClusterID Found.  A ClusterID is required for the cloud provider to function properly.  This check can be bypassed by setting the allow-untagged-cloud option")
-		}
-	}
-
-	if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
-		informerUserCloud.SetInformers(sharedInformers)
+		return ControllerContext{}, err
 	}
 
 	ctx := ControllerContext{
 		ClientBuilder:      clientBuilder,
 		InformerFactory:    sharedInformers,
-		Options:            *s,
+		ComponentConfig:    s.Generic.ComponentConfig,
 		AvailableResources: availableResources,
 		Cloud:              cloud,
+		LoopMode:           loopMode,
 		Stop:               stop,
 		InformersStarted:   make(chan struct{}),
+		ResyncPeriod:       ResyncPeriod(s),
 	}
 	return ctx, nil
 }
@@ -478,7 +429,7 @@ func StartControllers(ctx ControllerContext, startSATokenController InitFunc, co
 			continue
 		}
 
-		time.Sleep(wait.Jitter(ctx.Options.ControllerStartInterval.Duration, ControllerStartJitter))
+		time.Sleep(wait.Jitter(ctx.ComponentConfig.ControllerStartInterval.Duration, ControllerStartJitter))
 
 		glog.V(1).Infof("Starting %q", controllerName)
 		started, err := initFn(ctx)
@@ -509,23 +460,23 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 		return false, nil
 	}
 
-	if len(ctx.Options.ServiceAccountKeyFile) == 0 {
+	if len(ctx.ComponentConfig.ServiceAccountKeyFile) == 0 {
 		glog.Warningf("%q is disabled because there is no private key", saTokenControllerName)
 		return false, nil
 	}
-	privateKey, err := certutil.PrivateKeyFromFile(ctx.Options.ServiceAccountKeyFile)
+	privateKey, err := certutil.PrivateKeyFromFile(ctx.ComponentConfig.ServiceAccountKeyFile)
 	if err != nil {
 		return true, fmt.Errorf("error reading key for service account token controller: %v", err)
 	}
 
 	var rootCA []byte
-	if ctx.Options.RootCAFile != "" {
-		rootCA, err = ioutil.ReadFile(ctx.Options.RootCAFile)
+	if ctx.ComponentConfig.RootCAFile != "" {
+		rootCA, err = ioutil.ReadFile(ctx.ComponentConfig.RootCAFile)
 		if err != nil {
-			return true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.Options.RootCAFile, err)
+			return true, fmt.Errorf("error reading root-ca-file at %s: %v", ctx.ComponentConfig.RootCAFile, err)
 		}
 		if _, err := certutil.ParseCertsPEM(rootCA); err != nil {
-			return true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.Options.RootCAFile, err)
+			return true, fmt.Errorf("error parsing root-ca-file at %s: %v", ctx.ComponentConfig.RootCAFile, err)
 		}
 	} else {
 		rootCA = c.rootClientBuilder.ConfigOrDie("tokens-controller").CAData
@@ -536,14 +487,14 @@ func (c serviceAccountTokenControllerStarter) startServiceAccountTokenController
 		ctx.InformerFactory.Core().V1().Secrets(),
 		c.rootClientBuilder.ClientOrDie("tokens-controller"),
 		serviceaccountcontroller.TokensControllerOptions{
-			TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey),
+			TokenGenerator: serviceaccount.JWTTokenGenerator(serviceaccount.LegacyIssuer, privateKey),
 			RootCA:         rootCA,
 		},
 	)
 	if err != nil {
 		return true, fmt.Errorf("error creating Tokens controller: %v", err)
 	}
-	go controller.Run(int(ctx.Options.ConcurrentSATokenSyncs), ctx.Stop)
+	go controller.Run(int(ctx.ComponentConfig.ConcurrentSATokenSyncs), ctx.Stop)
 
 	// start the first set of informers now so that other controllers can start
 	ctx.InformerFactory.Start(ctx.Stop)

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/core.go

@@ -41,8 +41,9 @@ import (
 	endpointcontroller "k8s.io/kubernetes/pkg/controller/endpoint"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	namespacecontroller "k8s.io/kubernetes/pkg/controller/namespace"
-	nodecontroller "k8s.io/kubernetes/pkg/controller/node"
-	"k8s.io/kubernetes/pkg/controller/node/ipam"
+	nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
+	lifecyclecontroller "k8s.io/kubernetes/pkg/controller/nodelifecycle"
 	"k8s.io/kubernetes/pkg/controller/podgc"
 	replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
 	resourcequotacontroller "k8s.io/kubernetes/pkg/controller/resourcequota"
@@ -54,6 +55,7 @@ import (
 	"k8s.io/kubernetes/pkg/controller/volume/expand"
 	persistentvolumecontroller "k8s.io/kubernetes/pkg/controller/volume/persistentvolume"
 	"k8s.io/kubernetes/pkg/controller/volume/pvcprotection"
+	"k8s.io/kubernetes/pkg/controller/volume/pvprotection"
 	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/quota/generic"
 	quotainstall "k8s.io/kubernetes/pkg/quota/install"
@@ -66,70 +68,83 @@ func startServiceController(ctx ControllerContext) (bool, error) {
 		ctx.ClientBuilder.ClientOrDie("service-controller"),
 		ctx.InformerFactory.Core().V1().Services(),
 		ctx.InformerFactory.Core().V1().Nodes(),
-		ctx.Options.ClusterName,
+		ctx.ComponentConfig.ClusterName,
 	)
 	if err != nil {
 		// This error shouldn't fail. It lives like this as a legacy.
 		glog.Errorf("Failed to start service controller: %v", err)
 		return false, nil
 	}
-	go serviceController.Run(ctx.Stop, int(ctx.Options.ConcurrentServiceSyncs))
+	go serviceController.Run(ctx.Stop, int(ctx.ComponentConfig.ConcurrentServiceSyncs))
 	return true, nil
 }
 
-func startNodeController(ctx ControllerContext) (bool, error) {
+func startNodeIpamController(ctx ControllerContext) (bool, error) {
 	var clusterCIDR *net.IPNet = nil
 	var serviceCIDR *net.IPNet = nil
-	if ctx.Options.AllocateNodeCIDRs {
+	if ctx.ComponentConfig.AllocateNodeCIDRs {
 		var err error
-		if len(strings.TrimSpace(ctx.Options.ClusterCIDR)) != 0 {
-			_, clusterCIDR, err = net.ParseCIDR(ctx.Options.ClusterCIDR)
+		if len(strings.TrimSpace(ctx.ComponentConfig.ClusterCIDR)) != 0 {
+			_, clusterCIDR, err = net.ParseCIDR(ctx.ComponentConfig.ClusterCIDR)
 			if err != nil {
-				glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.Options.ClusterCIDR, err)
+				glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.ClusterCIDR, err)
 			}
 		}
 
-		if len(strings.TrimSpace(ctx.Options.ServiceCIDR)) != 0 {
-			_, serviceCIDR, err = net.ParseCIDR(ctx.Options.ServiceCIDR)
+		if len(strings.TrimSpace(ctx.ComponentConfig.ServiceCIDR)) != 0 {
+			_, serviceCIDR, err = net.ParseCIDR(ctx.ComponentConfig.ServiceCIDR)
 			if err != nil {
-				glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.Options.ServiceCIDR, err)
+				glog.Warningf("Unsuccessful parsing of service CIDR %v: %v", ctx.ComponentConfig.ServiceCIDR, err)
 			}
 		}
 	}
 
-	nodeController, err := nodecontroller.NewNodeController(
+	nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
+		ctx.InformerFactory.Core().V1().Nodes(),
+		ctx.Cloud,
+		ctx.ClientBuilder.ClientOrDie("node-controller"),
+		clusterCIDR,
+		serviceCIDR,
+		int(ctx.ComponentConfig.NodeCIDRMaskSize),
+		ctx.ComponentConfig.AllocateNodeCIDRs,
+		ipam.CIDRAllocatorType(ctx.ComponentConfig.CIDRAllocatorType),
+	)
+	if err != nil {
+		return true, err
+	}
+	go nodeIpamController.Run(ctx.Stop)
+	return true, nil
+}
+
+func startNodeLifecycleController(ctx ControllerContext) (bool, error) {
+	lifecycleController, err := lifecyclecontroller.NewNodeLifecycleController(
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.InformerFactory.Core().V1().Nodes(),
 		ctx.InformerFactory.Extensions().V1beta1().DaemonSets(),
 		ctx.Cloud,
 		ctx.ClientBuilder.ClientOrDie("node-controller"),
-		ctx.Options.PodEvictionTimeout.Duration,
-		ctx.Options.NodeEvictionRate,
-		ctx.Options.SecondaryNodeEvictionRate,
-		ctx.Options.LargeClusterSizeThreshold,
-		ctx.Options.UnhealthyZoneThreshold,
-		ctx.Options.NodeMonitorGracePeriod.Duration,
-		ctx.Options.NodeStartupGracePeriod.Duration,
-		ctx.Options.NodeMonitorPeriod.Duration,
-		clusterCIDR,
-		serviceCIDR,
-		int(ctx.Options.NodeCIDRMaskSize),
-		ctx.Options.AllocateNodeCIDRs,
-		ipam.CIDRAllocatorType(ctx.Options.CIDRAllocatorType),
-		ctx.Options.EnableTaintManager,
+		ctx.ComponentConfig.NodeMonitorPeriod.Duration,
+		ctx.ComponentConfig.NodeStartupGracePeriod.Duration,
+		ctx.ComponentConfig.NodeMonitorGracePeriod.Duration,
+		ctx.ComponentConfig.PodEvictionTimeout.Duration,
+		ctx.ComponentConfig.NodeEvictionRate,
+		ctx.ComponentConfig.SecondaryNodeEvictionRate,
+		ctx.ComponentConfig.LargeClusterSizeThreshold,
+		ctx.ComponentConfig.UnhealthyZoneThreshold,
+		ctx.ComponentConfig.EnableTaintManager,
 		utilfeature.DefaultFeatureGate.Enabled(features.TaintBasedEvictions),
 		utilfeature.DefaultFeatureGate.Enabled(features.TaintNodesByCondition),
 	)
 	if err != nil {
 		return true, err
 	}
-	go nodeController.Run(ctx.Stop)
+	go lifecycleController.Run(ctx.Stop)
 	return true, nil
 }
 
 func startRouteController(ctx ControllerContext) (bool, error) {
-	if !ctx.Options.AllocateNodeCIDRs || !ctx.Options.ConfigureCloudRoutes {
-		glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.Options.AllocateNodeCIDRs, ctx.Options.ConfigureCloudRoutes)
+	if !ctx.ComponentConfig.AllocateNodeCIDRs || !ctx.ComponentConfig.ConfigureCloudRoutes {
+		glog.Infof("Will not configure cloud provider routes for allocate-node-cidrs: %v, configure-cloud-routes: %v.", ctx.ComponentConfig.AllocateNodeCIDRs, ctx.ComponentConfig.ConfigureCloudRoutes)
 		return false, nil
 	}
 	if ctx.Cloud == nil {
@@ -141,26 +156,27 @@ func startRouteController(ctx ControllerContext) (bool, error) {
 		glog.Warning("configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
 		return false, nil
 	}
-	_, clusterCIDR, err := net.ParseCIDR(ctx.Options.ClusterCIDR)
+	_, clusterCIDR, err := net.ParseCIDR(ctx.ComponentConfig.ClusterCIDR)
 	if err != nil {
-		glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.Options.ClusterCIDR, err)
+		glog.Warningf("Unsuccessful parsing of cluster CIDR %v: %v", ctx.ComponentConfig.ClusterCIDR, err)
 	}
-	routeController := routecontroller.New(routes, ctx.ClientBuilder.ClientOrDie("route-controller"), ctx.InformerFactory.Core().V1().Nodes(), ctx.Options.ClusterName, clusterCIDR)
-	go routeController.Run(ctx.Stop, ctx.Options.RouteReconciliationPeriod.Duration)
+	routeController := routecontroller.New(routes, ctx.ClientBuilder.ClientOrDie("route-controller"), ctx.InformerFactory.Core().V1().Nodes(), ctx.ComponentConfig.ClusterName, clusterCIDR)
+	go routeController.Run(ctx.Stop, ctx.ComponentConfig.RouteReconciliationPeriod.Duration)
 	return true, nil
 }
 
 func startPersistentVolumeBinderController(ctx ControllerContext) (bool, error) {
 	params := persistentvolumecontroller.ControllerParameters{
 		KubeClient:                ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"),
-		SyncPeriod:                ctx.Options.PVClaimBinderSyncPeriod.Duration,
-		VolumePlugins:             ProbeControllerVolumePlugins(ctx.Cloud, ctx.Options.VolumeConfiguration),
+		SyncPeriod:                ctx.ComponentConfig.PVClaimBinderSyncPeriod.Duration,
+		VolumePlugins:             ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.VolumeConfiguration),
 		Cloud:                     ctx.Cloud,
-		ClusterName:               ctx.Options.ClusterName,
+		ClusterName:               ctx.ComponentConfig.ClusterName,
 		VolumeInformer:            ctx.InformerFactory.Core().V1().PersistentVolumes(),
 		ClaimInformer:             ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
 		ClassInformer:             ctx.InformerFactory.Storage().V1().StorageClasses(),
-		EnableDynamicProvisioning: ctx.Options.VolumeConfiguration.EnableDynamicProvisioning,
+		PodInformer:               ctx.InformerFactory.Core().V1().Pods(),
+		EnableDynamicProvisioning: ctx.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning,
 	}
 	volumeController, volumeControllerErr := persistentvolumecontroller.NewController(params)
 	if volumeControllerErr != nil {
@@ -171,7 +187,7 @@ func startPersistentVolumeBinderController(ctx ControllerContext) (bool, error)
 }
 
 func startAttachDetachController(ctx ControllerContext) (bool, error) {
-	if ctx.Options.ReconcilerSyncLoopPeriod.Duration < time.Second {
+	if ctx.ComponentConfig.ReconcilerSyncLoopPeriod.Duration < time.Second {
 		return true, fmt.Errorf("Duration time must be greater than one second as set via command line option reconcile-sync-loop-period.")
 	}
 	attachDetachController, attachDetachControllerErr :=
@@ -183,9 +199,9 @@ func startAttachDetachController(ctx ControllerContext) (bool, error) {
 			ctx.InformerFactory.Core().V1().PersistentVolumes(),
 			ctx.Cloud,
 			ProbeAttachableVolumePlugins(),
-			GetDynamicPluginProber(ctx.Options.VolumeConfiguration),
-			ctx.Options.DisableAttachDetachReconcilerSync,
-			ctx.Options.ReconcilerSyncLoopPeriod.Duration,
+			GetDynamicPluginProber(ctx.ComponentConfig.VolumeConfiguration),
+			ctx.ComponentConfig.DisableAttachDetachReconcilerSync,
+			ctx.ComponentConfig.ReconcilerSyncLoopPeriod.Duration,
 			attachdetach.DefaultTimerConfig,
 		)
 	if attachDetachControllerErr != nil {
@@ -202,7 +218,7 @@ func startVolumeExpandController(ctx ControllerContext) (bool, error) {
 			ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
 			ctx.InformerFactory.Core().V1().PersistentVolumes(),
 			ctx.Cloud,
-			ProbeExpandableVolumePlugins(ctx.Options.VolumeConfiguration))
+			ProbeExpandableVolumePlugins(ctx.ComponentConfig.VolumeConfiguration))
 
 		if expandControllerErr != nil {
 			return true, fmt.Errorf("Failed to start volume expand controller : %v", expandControllerErr)
@@ -219,7 +235,7 @@ func startEndpointController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().Services(),
 		ctx.InformerFactory.Core().V1().Endpoints(),
 		ctx.ClientBuilder.ClientOrDie("endpoint-controller"),
-	).Run(int(ctx.Options.ConcurrentEndpointSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentEndpointSyncs), ctx.Stop)
 	return true, nil
 }
 
@@ -229,7 +245,7 @@ func startReplicationController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().ReplicationControllers(),
 		ctx.ClientBuilder.ClientOrDie("replication-controller"),
 		replicationcontroller.BurstReplicas,
-	).Run(int(ctx.Options.ConcurrentRCSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentRCSyncs), ctx.Stop)
 	return true, nil
 }
 
@@ -237,7 +253,7 @@ func startPodGCController(ctx ControllerContext) (bool, error) {
 	go podgc.NewPodGC(
 		ctx.ClientBuilder.ClientOrDie("pod-garbage-collector"),
 		ctx.InformerFactory.Core().V1().Pods(),
-		int(ctx.Options.TerminatedPodGCThreshold),
+		int(ctx.ComponentConfig.TerminatedPodGCThreshold),
 	).Run(ctx.Stop)
 	return true, nil
 }
@@ -251,9 +267,9 @@ func startResourceQuotaController(ctx ControllerContext) (bool, error) {
 	resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{
 		QuotaClient:               resourceQuotaControllerClient.CoreV1(),
 		ResourceQuotaInformer:     ctx.InformerFactory.Core().V1().ResourceQuotas(),
-		ResyncPeriod:              controller.StaticResyncPeriodFunc(ctx.Options.ResourceQuotaSyncPeriod.Duration),
+		ResyncPeriod:              controller.StaticResyncPeriodFunc(ctx.ComponentConfig.ResourceQuotaSyncPeriod.Duration),
 		InformerFactory:           ctx.InformerFactory,
-		ReplenishmentResyncPeriod: ResyncPeriod(&ctx.Options),
+		ReplenishmentResyncPeriod: ctx.ResyncPeriod,
 		DiscoveryFunc:             discoveryFunc,
 		IgnoredResourcesFunc:      quotaConfiguration.IgnoredResources,
 		InformersStarted:          ctx.InformersStarted,
@@ -269,7 +285,7 @@ func startResourceQuotaController(ctx ControllerContext) (bool, error) {
 	if err != nil {
 		return false, err
 	}
-	go resourceQuotaController.Run(int(ctx.Options.ConcurrentResourceQuotaSyncs), ctx.Stop)
+	go resourceQuotaController.Run(int(ctx.ComponentConfig.ConcurrentResourceQuotaSyncs), ctx.Stop)
 
 	// Periodically the quota controller to detect new resource types
 	go resourceQuotaController.Sync(discoveryFunc, 30*time.Second, ctx.Stop)
@@ -297,10 +313,10 @@ func startNamespaceController(ctx ControllerContext) (bool, error) {
 		namespaceClientPool,
 		discoverResourcesFn,
 		ctx.InformerFactory.Core().V1().Namespaces(),
-		ctx.Options.NamespaceSyncPeriod.Duration,
+		ctx.ComponentConfig.NamespaceSyncPeriod.Duration,
 		v1.FinalizerKubernetes,
 	)
-	go namespaceController.Run(int(ctx.Options.ConcurrentNamespaceSyncs), ctx.Stop)
+	go namespaceController.Run(int(ctx.ComponentConfig.ConcurrentNamespaceSyncs), ctx.Stop)
 
 	return true, nil
 }
@@ -328,7 +344,7 @@ func startTTLController(ctx ControllerContext) (bool, error) {
 }
 
 func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
-	if !ctx.Options.EnableGarbageCollector {
+	if !ctx.ComponentConfig.EnableGarbageCollector {
 		return false, nil
 	}
 
@@ -344,14 +360,14 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
 	// TODO: Make NewMetadataCodecFactory support arbitrary (non-compiled)
 	// resource types. Otherwise we'll be storing full Unstructured data in our
 	// caches for custom resources. Consider porting it to work with
-	// metav1alpha1.PartialObjectMetadata.
+	// metav1beta1.PartialObjectMetadata.
 	metaOnlyClientPool := dynamic.NewClientPool(config, restMapper, dynamic.LegacyAPIPathResolverFunc)
 	clientPool := dynamic.NewClientPool(config, restMapper, dynamic.LegacyAPIPathResolverFunc)
 
 	// Get an initial set of deletable resources to prime the garbage collector.
 	deletableResources := garbagecollector.GetDeletableResources(discoveryClient)
 	ignoredResources := make(map[schema.GroupResource]struct{})
-	for _, r := range ctx.Options.GCIgnoredResources {
+	for _, r := range ctx.ComponentConfig.GCIgnoredResources {
 		ignoredResources[schema.GroupResource{Group: r.Group, Resource: r.Resource}] = struct{}{}
 	}
 	garbageCollector, err := garbagecollector.NewGarbageCollector(
@@ -368,7 +384,7 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
 	}
 
 	// Start the garbage collector.
-	workers := int(ctx.Options.ConcurrentGCSyncs)
+	workers := int(ctx.ComponentConfig.ConcurrentGCSyncs)
 	go garbageCollector.Run(workers, ctx.Stop)
 
 	// Periodically refresh the RESTMapper with new discovery information and sync
@@ -379,7 +395,7 @@ func startGarbageCollectorController(ctx ControllerContext) (bool, error) {
 }
 
 func startPVCProtectionController(ctx ControllerContext) (bool, error) {
-	if utilfeature.DefaultFeatureGate.Enabled(features.PVCProtection) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection) {
 		go pvcprotection.NewPVCProtectionController(
 			ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
 			ctx.InformerFactory.Core().V1().Pods(),
@@ -389,3 +405,14 @@ func startPVCProtectionController(ctx ControllerContext) (bool, error) {
 	}
 	return false, nil
 }
+
+func startPVProtectionController(ctx ControllerContext) (bool, error) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageObjectInUseProtection) {
+		go pvprotection.NewPVProtectionController(
+			ctx.InformerFactory.Core().V1().PersistentVolumes(),
+			ctx.ClientBuilder.ClientOrDie("pv-protection-controller"),
+		).Run(1, ctx.Stop)
+		return true, nil
+	}
+	return false, nil
+}

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/extensions.go

@@ -24,29 +24,10 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/kubernetes/pkg/controller/daemon"
 	"k8s.io/kubernetes/pkg/controller/deployment"
 	"k8s.io/kubernetes/pkg/controller/replicaset"
 )
 
-func startDaemonSetController(ctx ControllerContext) (bool, error) {
-	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "daemonsets"}] {
-		return false, nil
-	}
-	dsc, err := daemon.NewDaemonSetsController(
-		ctx.InformerFactory.Extensions().V1beta1().DaemonSets(),
-		ctx.InformerFactory.Apps().V1beta1().ControllerRevisions(),
-		ctx.InformerFactory.Core().V1().Pods(),
-		ctx.InformerFactory.Core().V1().Nodes(),
-		ctx.ClientBuilder.ClientOrDie("daemon-set-controller"),
-	)
-	if err != nil {
-		return true, fmt.Errorf("error creating DaemonSets controller: %v", err)
-	}
-	go dsc.Run(int(ctx.Options.ConcurrentDaemonSetSyncs), ctx.Stop)
-	return true, nil
-}
-
 func startDeploymentController(ctx ControllerContext) (bool, error) {
 	if !ctx.AvailableResources[schema.GroupVersionResource{Group: "extensions", Version: "v1beta1", Resource: "deployments"}] {
 		return false, nil
@@ -60,7 +41,7 @@ func startDeploymentController(ctx ControllerContext) (bool, error) {
 	if err != nil {
 		return true, fmt.Errorf("error creating Deployment controller: %v", err)
 	}
-	go dc.Run(int(ctx.Options.ConcurrentDeploymentSyncs), ctx.Stop)
+	go dc.Run(int(ctx.ComponentConfig.ConcurrentDeploymentSyncs), ctx.Stop)
 	return true, nil
 }
 
@@ -73,6 +54,6 @@ func startReplicaSetController(ctx ControllerContext) (bool, error) {
 		ctx.InformerFactory.Core().V1().Pods(),
 		ctx.ClientBuilder.ClientOrDie("replicaset-controller"),
 		replicaset.BurstReplicas,
-	).Run(int(ctx.Options.ConcurrentRSSyncs), ctx.Stop)
+	).Run(int(ctx.ComponentConfig.ConcurrentRSSyncs), ctx.Stop)
 	return true, nil
 }

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/options/BUILD

@@ -11,14 +11,14 @@ go_library(
     srcs = ["options.go"],
     importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options",
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
+        "//cmd/kube-controller-manager/app/config:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//pkg/client/leaderelectionconfig:go_default_library",
         "//pkg/controller/garbagecollector:go_default_library",
         "//pkg/features:go_default_library",
         "//pkg/master/ports:go_default_library",
-        "//vendor/github.com/cloudflare/cfssl/helpers:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
@@ -41,13 +41,14 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = ["options_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kube-controller-manager/app/options",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     tags = ["automanaged"],
     deps = [
+        "//cmd/controller-manager/app/options:go_default_library",
         "//pkg/apis/componentconfig:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
     ],
 )

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/options/options.go

@@ -21,12 +21,12 @@ package options
 import (
 	"fmt"
 	"strings"
-	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
+	kubecontrollerconfig "k8s.io/kubernetes/cmd/kube-controller-manager/app/config"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
@@ -35,213 +35,130 @@ import (
 	// add the kubernetes feature gates
 	_ "k8s.io/kubernetes/pkg/features"
 
-	"github.com/cloudflare/cfssl/helpers"
 	"github.com/spf13/pflag"
 )
 
-const (
-	// These defaults are deprecated and exported so that we can warn if
-	// they are being used.
-
-	// DefaultClusterSigningCertFile is deprecated. Do not use.
-	DefaultClusterSigningCertFile = "/etc/kubernetes/ca/ca.pem"
-	// DefaultClusterSigningKeyFile is deprecated. Do not use.
-	DefaultClusterSigningKeyFile = "/etc/kubernetes/ca/ca.key"
-)
-
-// CMServer is the main context object for the controller manager.
-type CMServer struct {
-	componentconfig.KubeControllerManagerConfiguration
-
-	Master     string
-	Kubeconfig string
+// KubeControllerManagerOptions is the main context object for the controller manager.
+type KubeControllerManagerOptions struct {
+	Generic cmoptions.GenericControllerManagerOptions
 }
 
-// NewCMServer creates a new CMServer with a default config.
-func NewCMServer() *CMServer {
+// NewKubeControllerManagerOptions creates a new KubeControllerManagerOptions with a default config.
+func NewKubeControllerManagerOptions() *KubeControllerManagerOptions {
+	componentConfig := cmoptions.NewDefaultControllerManagerComponentConfig(ports.InsecureKubeControllerManagerPort)
+	s := KubeControllerManagerOptions{
+		// The common/default are kept in 'cmd/kube-controller-manager/app/options/util.go'.
+		// Please make common changes there but put anything kube-controller specific here.
+		Generic: cmoptions.NewGenericControllerManagerOptions(componentConfig),
+	}
+
+	s.Generic.SecureServing.ServerCert.CertDirectory = "/var/run/kubernetes"
+	s.Generic.SecureServing.ServerCert.PairName = "kube-controller-manager"
+
 	gcIgnoredResources := make([]componentconfig.GroupResource, 0, len(garbagecollector.DefaultIgnoredResources()))
 	for r := range garbagecollector.DefaultIgnoredResources() {
 		gcIgnoredResources = append(gcIgnoredResources, componentconfig.GroupResource{Group: r.Group, Resource: r.Resource})
 	}
+	s.Generic.ComponentConfig.GCIgnoredResources = gcIgnoredResources
+	s.Generic.ComponentConfig.LeaderElection.LeaderElect = true
 
-	s := CMServer{
-		// Part of these default values also present in 'cmd/cloud-controller-manager/app/options/options.go'.
-		// Please keep them in sync when doing update.
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Controllers:                                     []string{"*"},
-			Port:                                            ports.ControllerManagerPort,
-			Address:                                         "0.0.0.0",
-			ConcurrentEndpointSyncs:                         5,
-			ConcurrentServiceSyncs:                          1,
-			ConcurrentRCSyncs:                               5,
-			ConcurrentRSSyncs:                               5,
-			ConcurrentDaemonSetSyncs:                        2,
-			ConcurrentJobSyncs:                              5,
-			ConcurrentResourceQuotaSyncs:                    5,
-			ConcurrentDeploymentSyncs:                       5,
-			ConcurrentNamespaceSyncs:                        10,
-			ConcurrentSATokenSyncs:                          5,
-			ServiceSyncPeriod:                               metav1.Duration{Duration: 5 * time.Minute},
-			RouteReconciliationPeriod:                       metav1.Duration{Duration: 10 * time.Second},
-			ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 5 * time.Minute},
-			NamespaceSyncPeriod:                             metav1.Duration{Duration: 5 * time.Minute},
-			PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 15 * time.Second},
-			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 30 * time.Second},
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 3 * time.Minute},
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 5 * time.Minute},
-			HorizontalPodAutoscalerTolerance:                0.1,
-			DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 30 * time.Second},
-			MinResyncPeriod:                                 metav1.Duration{Duration: 12 * time.Hour},
-			RegisterRetryCount:                              10,
-			PodEvictionTimeout:                              metav1.Duration{Duration: 5 * time.Minute},
-			NodeMonitorGracePeriod:                          metav1.Duration{Duration: 40 * time.Second},
-			NodeStartupGracePeriod:                          metav1.Duration{Duration: 60 * time.Second},
-			NodeMonitorPeriod:                               metav1.Duration{Duration: 5 * time.Second},
-			ClusterName:                                     "kubernetes",
-			NodeCIDRMaskSize:                                24,
-			ConfigureCloudRoutes:                            true,
-			TerminatedPodGCThreshold:                        12500,
-			VolumeConfiguration: componentconfig.VolumeConfiguration{
-				EnableHostPathProvisioning: false,
-				EnableDynamicProvisioning:  true,
-				PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
-					MaximumRetry:             3,
-					MinimumTimeoutNFS:        300,
-					IncrementTimeoutNFS:      30,
-					MinimumTimeoutHostPath:   60,
-					IncrementTimeoutHostPath: 30,
-				},
-				FlexVolumePluginDir: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec/",
-			},
-			ContentType:                           "application/vnd.kubernetes.protobuf",
-			KubeAPIQPS:                            20.0,
-			KubeAPIBurst:                          30,
-			LeaderElection:                        leaderelectionconfig.DefaultLeaderElectionConfiguration(),
-			ControllerStartInterval:               metav1.Duration{Duration: 0 * time.Second},
-			EnableGarbageCollector:                true,
-			ConcurrentGCSyncs:                     20,
-			GCIgnoredResources:                    gcIgnoredResources,
-			ClusterSigningCertFile:                DefaultClusterSigningCertFile,
-			ClusterSigningKeyFile:                 DefaultClusterSigningKeyFile,
-			ClusterSigningDuration:                metav1.Duration{Duration: helpers.OneYear},
-			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 60 * time.Second},
-			EnableTaintManager:                    true,
-			HorizontalPodAutoscalerUseRESTClients: true,
-		},
-	}
-	s.LeaderElection.LeaderElect = true
 	return &s
 }
 
-// AddFlags adds flags for a specific CMServer to the specified FlagSet
-func (s *CMServer) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) {
-	fs.StringSliceVar(&s.Controllers, "controllers", s.Controllers, fmt.Sprintf(""+
+// AddFlags adds flags for a specific KubeControllerManagerOptions to the specified FlagSet
+func (s *KubeControllerManagerOptions) AddFlags(fs *pflag.FlagSet, allControllers []string, disabledByDefaultControllers []string) {
+	s.Generic.AddFlags(fs)
+
+	fs.StringSliceVar(&s.Generic.ComponentConfig.Controllers, "controllers", s.Generic.ComponentConfig.Controllers, fmt.Sprintf(""+
 		"A list of controllers to enable.  '*' enables all on-by-default controllers, 'foo' enables the controller "+
 		"named 'foo', '-foo' disables the controller named 'foo'.\nAll controllers: %s\nDisabled-by-default controllers: %s",
 		strings.Join(allControllers, ", "), strings.Join(disabledByDefaultControllers, ", ")))
-	fs.Int32Var(&s.Port, "port", s.Port, "The port that the controller-manager's http service runs on")
-	fs.Var(componentconfig.IPVar{Val: &s.Address}, "address", "The IP address to serve on (set to 0.0.0.0 for all interfaces)")
-	fs.BoolVar(&s.UseServiceAccountCredentials, "use-service-account-credentials", s.UseServiceAccountCredentials, "If true, use individual service account credentials for each controller.")
-	fs.StringVar(&s.CloudProvider, "cloud-provider", s.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
-	fs.StringVar(&s.CloudConfigFile, "cloud-config", s.CloudConfigFile, "The path to the cloud provider configuration file.  Empty string for no configuration file.")
-	fs.BoolVar(&s.AllowUntaggedCloud, "allow-untagged-cloud", false, "Allow the cluster to run without the cluster-id on cloud instances.  This is a legacy mode of operation and a cluster-id will be required in the future.")
-	fs.MarkDeprecated("allow-untagged-cloud", "This flag is deprecated and will be removed in a future release.  A cluster-id will be required on cloud instances")
-	fs.Int32Var(&s.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentServiceSyncs, "concurrent-service-syncs", s.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentRCSyncs, "concurrent_rc_syncs", s.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load")
-	fs.Int32Var(&s.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", s.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load")
-	fs.DurationVar(&s.ServiceSyncPeriod.Duration, "service-sync-period", s.ServiceSyncPeriod.Duration, "The period for syncing services with their external load balancers")
-	fs.DurationVar(&s.NodeSyncPeriod.Duration, "node-sync-period", 0, ""+
+	fs.StringVar(&s.Generic.ComponentConfig.CloudProvider, "cloud-provider", s.Generic.ComponentConfig.CloudProvider, "The provider for cloud services.  Empty string for no provider.")
+	fs.StringVar(&s.Generic.ComponentConfig.ExternalCloudVolumePlugin, "external-cloud-volume-plugin", s.Generic.ComponentConfig.ExternalCloudVolumePlugin, "The plugin to use when cloud provider is set to external. Can be empty, should only be set when cloud-provider is external. Currently used to allow node and volume controllers to work for in tree cloud providers.")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentEndpointSyncs, "concurrent-endpoint-syncs", s.Generic.ComponentConfig.ConcurrentEndpointSyncs, "The number of endpoint syncing operations that will be done concurrently. Larger number = faster endpoint updating, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentServiceSyncs, "concurrent-service-syncs", s.Generic.ComponentConfig.ConcurrentServiceSyncs, "The number of services that are allowed to sync concurrently. Larger number = more responsive service management, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentRCSyncs, "concurrent_rc_syncs", s.Generic.ComponentConfig.ConcurrentRCSyncs, "The number of replication controllers that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentRSSyncs, "concurrent-replicaset-syncs", s.Generic.ComponentConfig.ConcurrentRSSyncs, "The number of replica sets that are allowed to sync concurrently. Larger number = more responsive replica management, but more CPU (and network) load")
+
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentResourceQuotaSyncs, "concurrent-resource-quota-syncs", s.Generic.ComponentConfig.ConcurrentResourceQuotaSyncs, "The number of resource quotas that are allowed to sync concurrently. Larger number = more responsive quota management, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentDeploymentSyncs, "concurrent-deployment-syncs", s.Generic.ComponentConfig.ConcurrentDeploymentSyncs, "The number of deployment objects that are allowed to sync concurrently. Larger number = more responsive deployments, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentNamespaceSyncs, "concurrent-namespace-syncs", s.Generic.ComponentConfig.ConcurrentNamespaceSyncs, "The number of namespace objects that are allowed to sync concurrently. Larger number = more responsive namespace termination, but more CPU (and network) load")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentSATokenSyncs, "concurrent-serviceaccount-token-syncs", s.Generic.ComponentConfig.ConcurrentSATokenSyncs, "The number of service account token objects that are allowed to sync concurrently. Larger number = more responsive token generation, but more CPU (and network) load")
+	fs.DurationVar(&s.Generic.ComponentConfig.NodeSyncPeriod.Duration, "node-sync-period", 0, ""+
 		"This flag is deprecated and will be removed in future releases. See node-monitor-period for Node health checking or "+
 		"route-reconciliation-period for cloud provider's route configuration settings.")
 	fs.MarkDeprecated("node-sync-period", "This flag is currently no-op and will be deleted.")
-	fs.DurationVar(&s.RouteReconciliationPeriod.Duration, "route-reconciliation-period", s.RouteReconciliationPeriod.Duration, "The period for reconciling routes created for Nodes by cloud provider.")
-	fs.DurationVar(&s.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system")
-	fs.DurationVar(&s.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates")
-	fs.DurationVar(&s.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims")
-	fs.DurationVar(&s.MinResyncPeriod.Duration, "min-resync-period", s.MinResyncPeriod.Duration, "The resync period in reflectors will be random between MinResyncPeriod and 2*MinResyncPeriod")
-	fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling")
-	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod")
-	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod")
-	fs.StringVar(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "pv-recycler-pod-template-filepath-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.")
-	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod.  This is for development and testing only and will not work in a multi-node cluster.")
-	fs.Int32Var(&s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", s.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod.  This is for development and testing only and will not work in a multi-node cluster.")
-	fs.BoolVar(&s.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", s.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features.  HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
-	fs.BoolVar(&s.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", s.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.")
-	fs.StringVar(&s.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
-	fs.Int32Var(&s.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
-	fs.DurationVar(&s.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
-	fs.DurationVar(&s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
-	fs.DurationVar(&s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
-	fs.Float64Var(&s.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", s.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
-	fs.DurationVar(&s.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
-	fs.DurationVar(&s.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
-	fs.Float32Var(&s.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
+	fs.DurationVar(&s.Generic.ComponentConfig.ResourceQuotaSyncPeriod.Duration, "resource-quota-sync-period", s.Generic.ComponentConfig.ResourceQuotaSyncPeriod.Duration, "The period for syncing quota usage status in the system")
+	fs.DurationVar(&s.Generic.ComponentConfig.NamespaceSyncPeriod.Duration, "namespace-sync-period", s.Generic.ComponentConfig.NamespaceSyncPeriod.Duration, "The period for syncing namespace life-cycle updates")
+	fs.DurationVar(&s.Generic.ComponentConfig.PVClaimBinderSyncPeriod.Duration, "pvclaimbinder-sync-period", s.Generic.ComponentConfig.PVClaimBinderSyncPeriod.Duration, "The period for syncing persistent volumes and persistent volume claims")
+	fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "pv-recycler-pod-template-filepath-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, "The file path to a pod definition used as a template for NFS persistent volume recycling")
+	fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "pv-recycler-minimum-timeout-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS, "The minimum ActiveDeadlineSeconds to use for an NFS Recycler pod")
+	fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "pv-recycler-increment-timeout-nfs", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS, "the increment of time added per Gi to ActiveDeadlineSeconds for an NFS scrubber pod")
+	fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "pv-recycler-pod-template-filepath-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, "The file path to a pod definition used as a template for HostPath persistent volume recycling. This is for development and testing only and will not work in a multi-node cluster.")
+	fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "pv-recycler-minimum-timeout-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath, "The minimum ActiveDeadlineSeconds to use for a HostPath Recycler pod.  This is for development and testing only and will not work in a multi-node cluster.")
+	fs.Int32Var(&s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "pv-recycler-timeout-increment-hostpath", s.Generic.ComponentConfig.VolumeConfiguration.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath, "the increment of time added per Gi to ActiveDeadlineSeconds for a HostPath scrubber pod.  This is for development and testing only and will not work in a multi-node cluster.")
+	fs.BoolVar(&s.Generic.ComponentConfig.VolumeConfiguration.EnableHostPathProvisioning, "enable-hostpath-provisioner", s.Generic.ComponentConfig.VolumeConfiguration.EnableHostPathProvisioning, "Enable HostPath PV provisioning when running without a cloud provider. This allows testing and development of provisioning features.  HostPath provisioning is not supported in any way, won't work in a multi-node cluster, and should not be used for anything other than testing or development.")
+	fs.BoolVar(&s.Generic.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning, "enable-dynamic-provisioning", s.Generic.ComponentConfig.VolumeConfiguration.EnableDynamicProvisioning, "Enable dynamic provisioning for environments that support it.")
+	fs.StringVar(&s.Generic.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir, "flex-volume-plugin-dir", s.Generic.ComponentConfig.VolumeConfiguration.FlexVolumePluginDir, "Full path of the directory in which the flex volume plugin should search for additional third party volume plugins.")
+	fs.Int32Var(&s.Generic.ComponentConfig.TerminatedPodGCThreshold, "terminated-pod-gc-threshold", s.Generic.ComponentConfig.TerminatedPodGCThreshold, "Number of terminated pods that can exist before the terminated pod garbage collector starts deleting terminated pods. If <= 0, the terminated pod garbage collector is disabled.")
+	fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration, "horizontal-pod-autoscaler-sync-period", s.Generic.ComponentConfig.HorizontalPodAutoscalerSyncPeriod.Duration, "The period for syncing the number of pods in horizontal pod autoscaler.")
+	fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-upscale-delay", s.Generic.ComponentConfig.HorizontalPodAutoscalerUpscaleForbiddenWindow.Duration, "The period since last upscale, before another upscale can be performed in horizontal pod autoscaler.")
+	fs.DurationVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "horizontal-pod-autoscaler-downscale-delay", s.Generic.ComponentConfig.HorizontalPodAutoscalerDownscaleForbiddenWindow.Duration, "The period since last downscale, before another downscale can be performed in horizontal pod autoscaler.")
+	fs.Float64Var(&s.Generic.ComponentConfig.HorizontalPodAutoscalerTolerance, "horizontal-pod-autoscaler-tolerance", s.Generic.ComponentConfig.HorizontalPodAutoscalerTolerance, "The minimum change (from 1.0) in the desired-to-actual metrics ratio for the horizontal pod autoscaler to consider scaling.")
+	fs.DurationVar(&s.Generic.ComponentConfig.DeploymentControllerSyncPeriod.Duration, "deployment-controller-sync-period", s.Generic.ComponentConfig.DeploymentControllerSyncPeriod.Duration, "Period for syncing the deployments.")
+	fs.DurationVar(&s.Generic.ComponentConfig.PodEvictionTimeout.Duration, "pod-eviction-timeout", s.Generic.ComponentConfig.PodEvictionTimeout.Duration, "The grace period for deleting pods on failed nodes.")
+	fs.Float32Var(&s.Generic.ComponentConfig.DeletingPodsQps, "deleting-pods-qps", 0.1, "Number of nodes per second on which pods are deleted in case of node failure.")
 	fs.MarkDeprecated("deleting-pods-qps", "This flag is currently no-op and will be deleted.")
-	fs.Int32Var(&s.DeletingPodsBurst, "deleting-pods-burst", 0, "Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.")
+	fs.Int32Var(&s.Generic.ComponentConfig.DeletingPodsBurst, "deleting-pods-burst", 0, "Number of nodes on which pods are bursty deleted in case of node failure. For more details look into RateLimiter.")
 	fs.MarkDeprecated("deleting-pods-burst", "This flag is currently no-op and will be deleted.")
-	fs.Int32Var(&s.RegisterRetryCount, "register-retry-count", s.RegisterRetryCount, ""+
+	fs.Int32Var(&s.Generic.ComponentConfig.RegisterRetryCount, "register-retry-count", s.Generic.ComponentConfig.RegisterRetryCount, ""+
 		"The number of retries for initial node registration.  Retry interval equals node-sync-period.")
 	fs.MarkDeprecated("register-retry-count", "This flag is currently no-op and will be deleted.")
-	fs.DurationVar(&s.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", s.NodeMonitorGracePeriod.Duration,
+	fs.DurationVar(&s.Generic.ComponentConfig.NodeMonitorGracePeriod.Duration, "node-monitor-grace-period", s.Generic.ComponentConfig.NodeMonitorGracePeriod.Duration,
 		"Amount of time which we allow running Node to be unresponsive before marking it unhealthy. "+
 			"Must be N times more than kubelet's nodeStatusUpdateFrequency, "+
 			"where N means number of retries allowed for kubelet to post node status.")
-	fs.DurationVar(&s.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.NodeStartupGracePeriod.Duration,
+	fs.DurationVar(&s.Generic.ComponentConfig.NodeStartupGracePeriod.Duration, "node-startup-grace-period", s.Generic.ComponentConfig.NodeStartupGracePeriod.Duration,
 		"Amount of time which we allow starting Node to be unresponsive before marking it unhealthy.")
-	fs.DurationVar(&s.NodeMonitorPeriod.Duration, "node-monitor-period", s.NodeMonitorPeriod.Duration,
-		"The period for syncing NodeStatus in NodeController.")
-	fs.StringVar(&s.ServiceAccountKeyFile, "service-account-private-key-file", s.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
-	fs.StringVar(&s.ClusterSigningCertFile, "cluster-signing-cert-file", s.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates")
-	fs.StringVar(&s.ClusterSigningKeyFile, "cluster-signing-key-file", s.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates")
-	fs.DurationVar(&s.ClusterSigningDuration.Duration, "experimental-cluster-signing-duration", s.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.")
+	fs.StringVar(&s.Generic.ComponentConfig.ServiceAccountKeyFile, "service-account-private-key-file", s.Generic.ComponentConfig.ServiceAccountKeyFile, "Filename containing a PEM-encoded private RSA or ECDSA key used to sign service account tokens.")
+	fs.StringVar(&s.Generic.ComponentConfig.ClusterSigningCertFile, "cluster-signing-cert-file", s.Generic.ComponentConfig.ClusterSigningCertFile, "Filename containing a PEM-encoded X509 CA certificate used to issue cluster-scoped certificates")
+	fs.StringVar(&s.Generic.ComponentConfig.ClusterSigningKeyFile, "cluster-signing-key-file", s.Generic.ComponentConfig.ClusterSigningKeyFile, "Filename containing a PEM-encoded RSA or ECDSA private key used to sign cluster-scoped certificates")
+	fs.DurationVar(&s.Generic.ComponentConfig.ClusterSigningDuration.Duration, "experimental-cluster-signing-duration", s.Generic.ComponentConfig.ClusterSigningDuration.Duration, "The length of duration signed certificates will be given.")
 	var dummy string
 	fs.MarkDeprecated("insecure-experimental-approve-all-kubelet-csrs-for-group", "This flag does nothing.")
 	fs.StringVar(&dummy, "insecure-experimental-approve-all-kubelet-csrs-for-group", "", "This flag does nothing.")
-	fs.BoolVar(&s.EnableProfiling, "profiling", true, "Enable profiling via web interface host:port/debug/pprof/")
-	fs.BoolVar(&s.EnableContentionProfiling, "contention-profiling", false, "Enable lock contention profiling, if profiling is enabled")
-	fs.StringVar(&s.ClusterName, "cluster-name", s.ClusterName, "The instance prefix for the cluster")
-	fs.StringVar(&s.ClusterCIDR, "cluster-cidr", s.ClusterCIDR, "CIDR Range for Pods in cluster. Requires --allocate-node-cidrs to be true")
-	fs.StringVar(&s.ServiceCIDR, "service-cluster-ip-range", s.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
-	fs.Int32Var(&s.NodeCIDRMaskSize, "node-cidr-mask-size", s.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
-	fs.BoolVar(&s.AllocateNodeCIDRs, "allocate-node-cidrs", false,
-		"Should CIDRs for Pods be allocated and set on the cloud provider.")
-	fs.StringVar(&s.CIDRAllocatorType, "cidr-allocator-type", "RangeAllocator",
-		"Type of CIDR allocator to use")
-	fs.BoolVar(&s.ConfigureCloudRoutes, "configure-cloud-routes", true, "Should CIDRs allocated by allocate-node-cidrs be configured on the cloud provider.")
-	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
-	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information.")
-	fs.StringVar(&s.RootCAFile, "root-ca-file", s.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.")
-	fs.StringVar(&s.ContentType, "kube-api-content-type", s.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&s.KubeAPIQPS, "kube-api-qps", s.KubeAPIQPS, "QPS to use while talking with kubernetes apiserver")
-	fs.Int32Var(&s.KubeAPIBurst, "kube-api-burst", s.KubeAPIBurst, "Burst to use while talking with kubernetes apiserver")
-	fs.DurationVar(&s.ControllerStartInterval.Duration, "controller-start-interval", s.ControllerStartInterval.Duration, "Interval between starting controller managers.")
-	fs.BoolVar(&s.EnableGarbageCollector, "enable-garbage-collector", s.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.")
-	fs.Int32Var(&s.ConcurrentGCSyncs, "concurrent-gc-syncs", s.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.")
-	fs.Float32Var(&s.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
-	fs.Float32Var(&s.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
-	fs.Int32Var(&s.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
-	fs.Float32Var(&s.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
-	fs.BoolVar(&s.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
-	fs.DurationVar(&s.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
-	fs.BoolVar(&s.EnableTaintManager, "enable-taint-manager", s.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
-	fs.BoolVar(&s.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature.  If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizontal pod autoscaler.")
+	fs.StringVar(&s.Generic.ComponentConfig.ServiceCIDR, "service-cluster-ip-range", s.Generic.ComponentConfig.ServiceCIDR, "CIDR Range for Services in cluster. Requires --allocate-node-cidrs to be true")
+	fs.Int32Var(&s.Generic.ComponentConfig.NodeCIDRMaskSize, "node-cidr-mask-size", s.Generic.ComponentConfig.NodeCIDRMaskSize, "Mask size for node cidr in cluster.")
+	fs.StringVar(&s.Generic.ComponentConfig.RootCAFile, "root-ca-file", s.Generic.ComponentConfig.RootCAFile, "If set, this root certificate authority will be included in service account's token secret. This must be a valid PEM-encoded CA bundle.")
+	fs.BoolVar(&s.Generic.ComponentConfig.EnableGarbageCollector, "enable-garbage-collector", s.Generic.ComponentConfig.EnableGarbageCollector, "Enables the generic garbage collector. MUST be synced with the corresponding flag of the kube-apiserver.")
+	fs.Int32Var(&s.Generic.ComponentConfig.ConcurrentGCSyncs, "concurrent-gc-syncs", s.Generic.ComponentConfig.ConcurrentGCSyncs, "The number of garbage collector workers that are allowed to sync concurrently.")
+	fs.Int32Var(&s.Generic.ComponentConfig.LargeClusterSizeThreshold, "large-cluster-size-threshold", 50, "Number of nodes from which NodeController treats the cluster as large for the eviction logic purposes. --secondary-node-eviction-rate is implicitly overridden to 0 for clusters this size or smaller.")
+	fs.Float32Var(&s.Generic.ComponentConfig.UnhealthyZoneThreshold, "unhealthy-zone-threshold", 0.55, "Fraction of Nodes in a zone which needs to be not Ready (minimum 3) for zone to be treated as unhealthy. ")
+	fs.BoolVar(&s.Generic.ComponentConfig.DisableAttachDetachReconcilerSync, "disable-attach-detach-reconcile-sync", false, "Disable volume attach detach reconciler sync. Disabling this may cause volumes to be mismatched with pods. Use wisely.")
+	fs.DurationVar(&s.Generic.ComponentConfig.ReconcilerSyncLoopPeriod.Duration, "attach-detach-reconcile-sync-period", s.Generic.ComponentConfig.ReconcilerSyncLoopPeriod.Duration, "The reconciler sync wait time between volume attach detach. This duration must be larger than one second, and increasing this value from the default may allow for volumes to be mismatched with pods.")
+	fs.BoolVar(&s.Generic.ComponentConfig.EnableTaintManager, "enable-taint-manager", s.Generic.ComponentConfig.EnableTaintManager, "WARNING: Beta feature. If set to true enables NoExecute Taints and will evict all not-tolerating Pod running on Nodes tainted with this kind of Taints.")
+	fs.BoolVar(&s.Generic.ComponentConfig.HorizontalPodAutoscalerUseRESTClients, "horizontal-pod-autoscaler-use-rest-clients", s.Generic.ComponentConfig.HorizontalPodAutoscalerUseRESTClients, "WARNING: alpha feature.  If set to true, causes the horizontal pod autoscaler controller to use REST clients through the kube-aggregator, instead of using the legacy metrics client through the API server proxy.  This is required for custom metrics support in the horizontal pod autoscaler.")
+	fs.Float32Var(&s.Generic.ComponentConfig.NodeEvictionRate, "node-eviction-rate", 0.1, "Number of nodes per second on which pods are deleted in case of node failure when a zone is healthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters.")
+	fs.Float32Var(&s.Generic.ComponentConfig.SecondaryNodeEvictionRate, "secondary-node-eviction-rate", 0.01, "Number of nodes per second on which pods are deleted in case of node failure when a zone is unhealthy (see --unhealthy-zone-threshold for definition of healthy/unhealthy). Zone refers to entire cluster in non-multizone clusters. This value is implicitly overridden to 0 if the cluster size is smaller than --large-cluster-size-threshold.")
 
-	leaderelectionconfig.BindFlags(&s.LeaderElection, fs)
+	leaderelectionconfig.BindFlags(&s.Generic.ComponentConfig.LeaderElection, fs)
 
 	utilfeature.DefaultFeatureGate.AddFlag(fs)
 }
 
+// ApplyTo fills up controller manager config with options.
+func (s *KubeControllerManagerOptions) ApplyTo(c *kubecontrollerconfig.Config) error {
+	err := s.Generic.ApplyTo(&c.Generic, "controller-manager")
+
+	return err
+}
+
 // Validate is used to validate the options and config before launching the controller manager
-func (s *CMServer) Validate(allControllers []string, disabledByDefaultControllers []string) error {
+func (s *KubeControllerManagerOptions) Validate(allControllers []string, disabledByDefaultControllers []string) error {
 	var errs []error
 
 	allControllersSet := sets.NewString(allControllers...)
-	for _, controller := range s.Controllers {
+	for _, controller := range s.Generic.ComponentConfig.Controllers {
 		if controller == "*" {
 			continue
 		}
@@ -256,3 +173,17 @@ func (s *CMServer) Validate(allControllers []string, disabledByDefaultController
 
 	return utilerrors.NewAggregate(errs)
 }
+
+// Config return a controller manager config objective
+func (s KubeControllerManagerOptions) Config(allControllers []string, disabledByDefaultControllers []string) (*kubecontrollerconfig.Config, error) {
+	if err := s.Validate(allControllers, disabledByDefaultControllers); err != nil {
+		return nil, err
+	}
+
+	c := &kubecontrollerconfig.Config{}
+	if err := s.ApplyTo(c); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/options/options_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"net"
 	"reflect"
 	"sort"
 	"testing"
@@ -26,12 +27,14 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	apiserveroptions "k8s.io/apiserver/pkg/server/options"
+	cmoptions "k8s.io/kubernetes/cmd/controller-manager/app/options"
 	"k8s.io/kubernetes/pkg/apis/componentconfig"
 )
 
 func TestAddFlags(t *testing.T) {
 	f := pflag.NewFlagSet("addflagstest", pflag.ContinueOnError)
-	s := NewCMServer()
+	s := NewKubeControllerManagerOptions()
 	s.AddFlags(f, []string{""}, []string{""})
 
 	args := []string{
@@ -69,6 +72,7 @@ func TestAddFlags(t *testing.T) {
 		"--horizontal-pod-autoscaler-downscale-delay=2m",
 		"--horizontal-pod-autoscaler-sync-period=45s",
 		"--horizontal-pod-autoscaler-upscale-delay=1m",
+		"--http2-max-streams-per-connection=47",
 		"--kube-api-burst=100",
 		"--kube-api-content-type=application/json",
 		"--kube-api-qps=50.0",
@@ -99,118 +103,135 @@ func TestAddFlags(t *testing.T) {
 		"--route-reconciliation-period=30s",
 		"--secondary-node-eviction-rate=0.05",
 		"--service-account-private-key-file=/service-account-private-key",
-		"--service-sync-period=2m",
 		"--terminated-pod-gc-threshold=12000",
 		"--unhealthy-zone-threshold=0.6",
 		"--use-service-account-credentials=true",
+		"--cert-dir=/a/b/c",
+		"--bind-address=192.168.4.21",
+		"--secure-port=10001",
 	}
 	f.Parse(args)
 	// Sort GCIgnoredResources because it's built from a map, which means the
 	// insertion order is random.
-	sort.Sort(sortedGCIgnoredResources(s.GCIgnoredResources))
+	sort.Sort(sortedGCIgnoredResources(s.Generic.ComponentConfig.GCIgnoredResources))
 
-	expected := &CMServer{
-		KubeControllerManagerConfiguration: componentconfig.KubeControllerManagerConfiguration{
-			Port:                                            10000,
-			Address:                                         "192.168.4.10",
-			AllocateNodeCIDRs:                               true,
-			CloudConfigFile:                                 "/cloud-config",
-			CloudProvider:                                   "gce",
-			ClusterCIDR:                                     "1.2.3.4/24",
-			ClusterName:                                     "k8s",
-			ConcurrentDeploymentSyncs:                       10,
-			ConcurrentEndpointSyncs:                         10,
-			ConcurrentGCSyncs:                               30,
-			ConcurrentNamespaceSyncs:                        20,
-			ConcurrentRSSyncs:                               10,
-			ConcurrentResourceQuotaSyncs:                    10,
-			ConcurrentServiceSyncs:                          2,
-			ConcurrentSATokenSyncs:                          10,
-			ConcurrentRCSyncs:                               10,
-			ConfigureCloudRoutes:                            false,
-			EnableContentionProfiling:                       true,
-			ControllerStartInterval:                         metav1.Duration{Duration: 2 * time.Minute},
-			ConcurrentDaemonSetSyncs:                        2,
-			ConcurrentJobSyncs:                              5,
-			DeletingPodsQps:                                 0.1,
-			EnableProfiling:                                 false,
-			CIDRAllocatorType:                               "CloudAllocator",
-			NodeCIDRMaskSize:                                48,
-			ServiceSyncPeriod:                               metav1.Duration{Duration: 2 * time.Minute},
-			ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 10 * time.Minute},
-			NamespaceSyncPeriod:                             metav1.Duration{Duration: 10 * time.Minute},
-			PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 30 * time.Second},
-			HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 45 * time.Second},
-			DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 45 * time.Second},
-			MinResyncPeriod:                                 metav1.Duration{Duration: 8 * time.Hour},
-			RegisterRetryCount:                              10,
-			RouteReconciliationPeriod:                       metav1.Duration{Duration: 30 * time.Second},
-			PodEvictionTimeout:                              metav1.Duration{Duration: 2 * time.Minute},
-			NodeMonitorGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
-			NodeStartupGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
-			NodeMonitorPeriod:                               metav1.Duration{Duration: 10 * time.Second},
-			HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 1 * time.Minute},
-			HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
-			HorizontalPodAutoscalerTolerance:                0.1,
-			TerminatedPodGCThreshold:                        12000,
-			VolumeConfiguration: componentconfig.VolumeConfiguration{
-				EnableDynamicProvisioning:  false,
-				EnableHostPathProvisioning: true,
-				FlexVolumePluginDir:        "/flex-volume-plugin",
-				PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
-					MaximumRetry:             3,
-					MinimumTimeoutNFS:        200,
-					IncrementTimeoutNFS:      45,
-					MinimumTimeoutHostPath:   45,
-					IncrementTimeoutHostPath: 45,
+	expected := &KubeControllerManagerOptions{
+		Generic: cmoptions.GenericControllerManagerOptions{
+			ComponentConfig: componentconfig.KubeControllerManagerConfiguration{
+				Port:                                            10252,     // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				Address:                                         "0.0.0.0", // Note: InsecureServingOptions.ApplyTo will write the flag value back into the component config
+				AllocateNodeCIDRs:                               true,
+				CloudConfigFile:                                 "/cloud-config",
+				CloudProvider:                                   "gce",
+				ClusterCIDR:                                     "1.2.3.4/24",
+				ClusterName:                                     "k8s",
+				ConcurrentDeploymentSyncs:                       10,
+				ConcurrentEndpointSyncs:                         10,
+				ConcurrentGCSyncs:                               30,
+				ConcurrentNamespaceSyncs:                        20,
+				ConcurrentRSSyncs:                               10,
+				ConcurrentResourceQuotaSyncs:                    10,
+				ConcurrentServiceSyncs:                          2,
+				ConcurrentSATokenSyncs:                          10,
+				ConcurrentRCSyncs:                               10,
+				ConfigureCloudRoutes:                            false,
+				EnableContentionProfiling:                       true,
+				ControllerStartInterval:                         metav1.Duration{Duration: 2 * time.Minute},
+				ConcurrentDaemonSetSyncs:                        2,
+				ConcurrentJobSyncs:                              5,
+				DeletingPodsQps:                                 0.1,
+				EnableProfiling:                                 false,
+				CIDRAllocatorType:                               "CloudAllocator",
+				NodeCIDRMaskSize:                                48,
+				ResourceQuotaSyncPeriod:                         metav1.Duration{Duration: 10 * time.Minute},
+				NamespaceSyncPeriod:                             metav1.Duration{Duration: 10 * time.Minute},
+				PVClaimBinderSyncPeriod:                         metav1.Duration{Duration: 30 * time.Second},
+				HorizontalPodAutoscalerSyncPeriod:               metav1.Duration{Duration: 45 * time.Second},
+				DeploymentControllerSyncPeriod:                  metav1.Duration{Duration: 45 * time.Second},
+				MinResyncPeriod:                                 metav1.Duration{Duration: 8 * time.Hour},
+				RegisterRetryCount:                              10,
+				RouteReconciliationPeriod:                       metav1.Duration{Duration: 30 * time.Second},
+				PodEvictionTimeout:                              metav1.Duration{Duration: 2 * time.Minute},
+				NodeMonitorGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
+				NodeStartupGracePeriod:                          metav1.Duration{Duration: 30 * time.Second},
+				NodeMonitorPeriod:                               metav1.Duration{Duration: 10 * time.Second},
+				HorizontalPodAutoscalerUpscaleForbiddenWindow:   metav1.Duration{Duration: 1 * time.Minute},
+				HorizontalPodAutoscalerDownscaleForbiddenWindow: metav1.Duration{Duration: 2 * time.Minute},
+				HorizontalPodAutoscalerTolerance:                0.1,
+				TerminatedPodGCThreshold:                        12000,
+				VolumeConfiguration: componentconfig.VolumeConfiguration{
+					EnableDynamicProvisioning:  false,
+					EnableHostPathProvisioning: true,
+					FlexVolumePluginDir:        "/flex-volume-plugin",
+					PersistentVolumeRecyclerConfiguration: componentconfig.PersistentVolumeRecyclerConfiguration{
+						MaximumRetry:             3,
+						MinimumTimeoutNFS:        200,
+						IncrementTimeoutNFS:      45,
+						MinimumTimeoutHostPath:   45,
+						IncrementTimeoutHostPath: 45,
+					},
 				},
+				ContentType:  "application/json",
+				KubeAPIQPS:   50.0,
+				KubeAPIBurst: 100,
+				LeaderElection: componentconfig.LeaderElectionConfiguration{
+					ResourceLock:  "configmap",
+					LeaderElect:   false,
+					LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
+					RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
+					RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+				},
+				ClusterSigningCertFile: "/cluster-signing-cert",
+				ClusterSigningKeyFile:  "/cluster-signing-key",
+				ServiceAccountKeyFile:  "/service-account-private-key",
+				ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour},
+				EnableGarbageCollector: false,
+				GCIgnoredResources: []componentconfig.GroupResource{
+					{Group: "extensions", Resource: "replicationcontrollers"},
+					{Group: "", Resource: "bindings"},
+					{Group: "", Resource: "componentstatuses"},
+					{Group: "", Resource: "events"},
+					{Group: "authentication.k8s.io", Resource: "tokenreviews"},
+					{Group: "authorization.k8s.io", Resource: "subjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"},
+					{Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"},
+					{Group: "apiregistration.k8s.io", Resource: "apiservices"},
+					{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"},
+				},
+				NodeEvictionRate:                      0.2,
+				SecondaryNodeEvictionRate:             0.05,
+				LargeClusterSizeThreshold:             100,
+				UnhealthyZoneThreshold:                0.6,
+				DisableAttachDetachReconcilerSync:     true,
+				ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 30 * time.Second},
+				Controllers:                           []string{"foo", "bar"},
+				EnableTaintManager:                    false,
+				HorizontalPodAutoscalerUseRESTClients: true,
+				UseServiceAccountCredentials:          true,
 			},
-			ContentType:  "application/json",
-			KubeAPIQPS:   50.0,
-			KubeAPIBurst: 100,
-			LeaderElection: componentconfig.LeaderElectionConfiguration{
-				ResourceLock:  "configmap",
-				LeaderElect:   false,
-				LeaseDuration: metav1.Duration{Duration: 30 * time.Second},
-				RenewDeadline: metav1.Duration{Duration: 15 * time.Second},
-				RetryPeriod:   metav1.Duration{Duration: 5 * time.Second},
+			SecureServing: &apiserveroptions.SecureServingOptions{
+				BindPort:    10001,
+				BindAddress: net.ParseIP("192.168.4.21"),
+				ServerCert: apiserveroptions.GeneratableKeyCert{
+					CertDirectory: "/a/b/c",
+					PairName:      "kube-controller-manager",
+				},
+				HTTP2MaxStreamsPerConnection: 47,
 			},
-			ClusterSigningCertFile: "/cluster-signing-cert",
-			ClusterSigningKeyFile:  "/cluster-signing-key",
-			ServiceAccountKeyFile:  "/service-account-private-key",
-			ClusterSigningDuration: metav1.Duration{Duration: 10 * time.Hour},
-			EnableGarbageCollector: false,
-			GCIgnoredResources: []componentconfig.GroupResource{
-				{Group: "extensions", Resource: "replicationcontrollers"},
-				{Group: "", Resource: "bindings"},
-				{Group: "", Resource: "componentstatuses"},
-				{Group: "", Resource: "events"},
-				{Group: "authentication.k8s.io", Resource: "tokenreviews"},
-				{Group: "authorization.k8s.io", Resource: "subjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "selfsubjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "localsubjectaccessreviews"},
-				{Group: "authorization.k8s.io", Resource: "selfsubjectrulesreviews"},
-				{Group: "apiregistration.k8s.io", Resource: "apiservices"},
-				{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"},
+			InsecureServing: &cmoptions.InsecureServingOptions{
+				BindAddress: net.ParseIP("192.168.4.10"),
+				BindPort:    int(10000),
+				BindNetwork: "tcp",
 			},
-			NodeEvictionRate:                      0.2,
-			SecondaryNodeEvictionRate:             0.05,
-			LargeClusterSizeThreshold:             100,
-			UnhealthyZoneThreshold:                0.6,
-			DisableAttachDetachReconcilerSync:     true,
-			ReconcilerSyncLoopPeriod:              metav1.Duration{Duration: 30 * time.Second},
-			Controllers:                           []string{"foo", "bar"},
-			EnableTaintManager:                    false,
-			HorizontalPodAutoscalerUseRESTClients: true,
-			UseServiceAccountCredentials:          true,
+			Kubeconfig: "/kubeconfig",
+			Master:     "192.168.4.20",
 		},
-		Kubeconfig: "/kubeconfig",
-		Master:     "192.168.4.20",
 	}
 
 	// Sort GCIgnoredResources because it's built from a map, which means the
 	// insertion order is random.
-	sort.Sort(sortedGCIgnoredResources(expected.GCIgnoredResources))
+	sort.Sort(sortedGCIgnoredResources(expected.Generic.ComponentConfig.GCIgnoredResources))
 
 	if !reflect.DeepEqual(expected, s) {
 		t.Errorf("Got different run options than expected.\nDifference detected on:\n%s", diff.ObjectReflectDiff(expected, s))

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/app/plugins.go

@@ -102,13 +102,11 @@ func ProbeExpandableVolumePlugins(config componentconfig.VolumeConfiguration) []
 	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
+	allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, photon_pd.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
 	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
-	if !utilfeature.DefaultFeatureGate.Enabled(features.CSIPersistentVolume) {
-		allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
-	}
 	return allPlugins
 }
 

vendor/k8s.io/kubernetes/cmd/kube-controller-manager/controller-manager.go

@@ -21,33 +21,38 @@ limitations under the License.
 package main
 
 import (
+	goflag "flag"
 	"fmt"
+	"math/rand"
 	"os"
+	"time"
 
-	"k8s.io/apiserver/pkg/util/flag"
+	"github.com/spf13/pflag"
+
+	utilflag "k8s.io/apiserver/pkg/util/flag"
 	"k8s.io/apiserver/pkg/util/logs"
 	"k8s.io/kubernetes/cmd/kube-controller-manager/app"
-	"k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
 	_ "k8s.io/kubernetes/pkg/util/reflector/prometheus" // for reflector metric registration
 	_ "k8s.io/kubernetes/pkg/util/workqueue/prometheus" // for workqueue metric registration
 	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
-	"k8s.io/kubernetes/pkg/version/verflag"
-
-	"github.com/spf13/pflag"
 )
 
 func main() {
-	s := options.NewCMServer()
-	s.AddFlags(pflag.CommandLine, app.KnownControllers(), app.ControllersDisabledByDefault.List())
+	rand.Seed(time.Now().UTC().UnixNano())
 
-	flag.InitFlags()
+	command := app.NewControllerManagerCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	verflag.PrintAndExitIfRequested()
-
-	if err := app.Run(s); err != nil {
+	if err := command.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}

vendor/k8s.io/kubernetes/cmd/kube-proxy/BUILD

@@ -9,14 +9,8 @@ load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "kube-proxy",
-    gc_linkopts = [
-        "-linkmode",
-        "external",
-        "-extldflags",
-        "-static",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/kube-proxy",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
+    pure = "on",
     x_defs = version_x_defs(),
 )
 

vendor/k8s.io/kubernetes/cmd/kube-proxy/app/BUILD

@@ -11,9 +11,38 @@ go_library(
     srcs = [
         "conntrack.go",
         "server.go",
-        "server_others.go",
     ] + select({
-        "@io_bazel_rules_go//go/platform:windows_amd64": [
+        "@io_bazel_rules_go//go/platform:android": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "server_others.go",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
             "server_windows.go",
         ],
         "//conditions:default": [],
@@ -24,7 +53,6 @@ go_library(
         "//pkg/apis/core:go_default_library",
         "//pkg/client/clientset_generated/internalclientset:go_default_library",
         "//pkg/client/informers/informers_generated/internalversion:go_default_library",
-        "//pkg/features:go_default_library",
         "//pkg/kubectl/cmd/util:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
         "//pkg/master/ports:go_default_library",
@@ -37,10 +65,9 @@ go_library(
         "//pkg/proxy/healthcheck:go_default_library",
         "//pkg/proxy/iptables:go_default_library",
         "//pkg/proxy/ipvs:go_default_library",
-        "//pkg/proxy/metrics:go_default_library",
         "//pkg/proxy/userspace:go_default_library",
         "//pkg/util/configz:go_default_library",
-        "//pkg/util/dbus:go_default_library",
+        "//pkg/util/flag:go_default_library",
         "//pkg/util/ipset:go_default_library",
         "//pkg/util/iptables:go_default_library",
         "//pkg/util/ipvs:go_default_library",
@@ -61,12 +88,13 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/serializer/json:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
-        "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/mux:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/routes:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",
@@ -75,10 +103,82 @@ go_library(
         "//vendor/k8s.io/client-go/tools/record:go_default_library",
         "//vendor/k8s.io/utils/exec:go_default_library",
     ] + select({
-        "@io_bazel_rules_go//go/platform:windows_amd64": [
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//pkg/features:go_default_library",
+            "//pkg/proxy/metrics:go_default_library",
+            "//pkg/util/dbus:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:windows": [
             "//pkg/proxy/winkernel:go_default_library",
             "//pkg/proxy/winuserspace:go_default_library",
             "//pkg/util/netsh:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/types:go_default_library",
+            "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library",
         ],
         "//conditions:default": [],
     }),
@@ -86,19 +186,104 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["server_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kube-proxy/app",
-    library = ":go_default_library",
+    srcs = [
+        "server_test.go",
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "server_others_test.go",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "server_others_test.go",
+        ],
+        "//conditions:default": [],
+    }),
+    embed = [":go_default_library"],
     deps = [
         "//pkg/apis/core:go_default_library",
+        "//pkg/features:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig:go_default_library",
         "//pkg/util/configz:go_default_library",
-        "//pkg/util/iptables:go_default_library",
         "//pkg/util/pointer:go_default_library",
         "//vendor/github.com/stretchr/testify/assert:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-    ],
+    ] + select({
+        "@io_bazel_rules_go//go/platform:android": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:darwin": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:dragonfly": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:freebsd": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:linux": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:nacl": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:netbsd": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:openbsd": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:plan9": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "@io_bazel_rules_go//go/platform:solaris": [
+            "//pkg/proxy/ipvs:go_default_library",
+            "//pkg/util/iptables:go_default_library",
+            "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(

vendor/k8s.io/kubernetes/cmd/kube-proxy/app/server.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
-	"net/http/pprof"
 	"os"
 	goruntime "runtime"
 	"strings"
@@ -38,7 +37,10 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/mux"
+	"k8s.io/apiserver/pkg/server/routes"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/util/flag"
 	clientgoclientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/rest"
@@ -63,6 +65,7 @@ import (
 	"k8s.io/kubernetes/pkg/proxy/ipvs"
 	"k8s.io/kubernetes/pkg/proxy/userspace"
 	"k8s.io/kubernetes/pkg/util/configz"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
 	utilipset "k8s.io/kubernetes/pkg/util/ipset"
 	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
 	utilipvs "k8s.io/kubernetes/pkg/util/ipvs"
@@ -115,58 +118,61 @@ type Options struct {
 }
 
 // AddFlags adds flags to fs and binds them to options.
-func AddFlags(options *Options, fs *pflag.FlagSet) {
-	fs.StringVar(&options.ConfigFile, "config", options.ConfigFile, "The path to the configuration file.")
-	fs.StringVar(&options.WriteConfigTo, "write-config-to", options.WriteConfigTo, "If set, write the default configuration values to this file and exit.")
-	fs.BoolVar(&options.CleanupAndExit, "cleanup-iptables", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.")
+	fs.StringVar(&o.WriteConfigTo, "write-config-to", o.WriteConfigTo, "If set, write the default configuration values to this file and exit.")
+	fs.BoolVar(&o.CleanupAndExit, "cleanup-iptables", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
 	fs.MarkDeprecated("cleanup-iptables", "This flag is replaced by --cleanup.")
-	fs.BoolVar(&options.CleanupAndExit, "cleanup", options.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
-	fs.BoolVar(&options.CleanupIPVS, "cleanup-ipvs", options.CleanupIPVS, "If true make kube-proxy cleanup ipvs rules before running.  Default is true")
+	fs.BoolVar(&o.CleanupAndExit, "cleanup", o.CleanupAndExit, "If true cleanup iptables and ipvs rules and exit.")
+	fs.BoolVar(&o.CleanupIPVS, "cleanup-ipvs", o.CleanupIPVS, "If true make kube-proxy cleanup ipvs rules before running.  Default is true")
 
 	// All flags below here are deprecated and will eventually be removed.
 
-	fs.Var(componentconfig.IPVar{Val: &options.config.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
-	fs.StringVar(&options.master, "master", options.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
-	fs.Int32Var(&options.healthzPort, "healthz-port", options.healthzPort, "The port to bind the health check server. Use 0 to disable.")
-	fs.Var(componentconfig.IPVar{Val: &options.config.HealthzBindAddress}, "healthz-bind-address", "The IP address and port for the health check server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
-	fs.Var(componentconfig.IPVar{Val: &options.config.MetricsBindAddress}, "metrics-bind-address", "The IP address and port for the metrics server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
-	fs.Int32Var(options.config.OOMScoreAdj, "oom-score-adj", utilpointer.Int32PtrDerefOr(options.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]")
-	fs.StringVar(&options.config.ResourceContainer, "resource-container", options.config.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).")
+	fs.Var(componentconfig.IPVar{Val: &o.config.BindAddress}, "bind-address", "The IP address for the proxy server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
+	fs.StringVar(&o.master, "master", o.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
+	fs.Int32Var(&o.healthzPort, "healthz-port", o.healthzPort, "The port to bind the health check server. Use 0 to disable.")
+	fs.Var(componentconfig.IPVar{Val: &o.config.HealthzBindAddress}, "healthz-bind-address", "The IP address and port for the health check server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
+	fs.Var(componentconfig.IPVar{Val: &o.config.MetricsBindAddress}, "metrics-bind-address", "The IP address and port for the metrics server to serve on (set to `0.0.0.0` for all IPv4 interfaces and `::` for all IPv6 interfaces)")
+	fs.Int32Var(o.config.OOMScoreAdj, "oom-score-adj", utilpointer.Int32PtrDerefOr(o.config.OOMScoreAdj, int32(qos.KubeProxyOOMScoreAdj)), "The oom-score-adj value for kube-proxy process. Values must be within the range [-1000, 1000]")
+	fs.StringVar(&o.config.ResourceContainer, "resource-container", o.config.ResourceContainer, "Absolute name of the resource-only container to create and run the Kube-proxy in (Default: /kube-proxy).")
 	fs.MarkDeprecated("resource-container", "This feature will be removed in a later release.")
-	fs.StringVar(&options.config.ClientConnection.KubeConfigFile, "kubeconfig", options.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
-	fs.Var(componentconfig.PortRangeVar{Val: &options.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.")
-	fs.StringVar(&options.config.HostnameOverride, "hostname-override", options.config.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
-	fs.Var(&options.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs'(experimental)'. If blank, use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
-	fs.Int32Var(options.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", utilpointer.Int32PtrDerefOr(options.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.  Must be within the range [0, 31].")
-	fs.DurationVar(&options.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", options.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
-	fs.DurationVar(&options.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", options.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
-	fs.DurationVar(&options.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", options.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
-	fs.DurationVar(&options.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", options.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
-	fs.DurationVar(&options.config.ConfigSyncPeriod.Duration, "config-sync-period", options.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed.  Must be greater than 0.")
-	fs.BoolVar(&options.config.IPTables.MasqueradeAll, "masquerade-all", options.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)")
-	fs.StringVar(&options.config.ClusterCIDR, "cluster-cidr", options.config.ClusterCIDR, "The CIDR range of pods in the cluster. When configured, traffic sent to a Service cluster IP from outside this range will be masqueraded and traffic sent from pods to an external LoadBalancer IP will be directed to the respective cluster IP instead")
-	fs.StringVar(&options.config.ClientConnection.ContentType, "kube-api-content-type", options.config.ClientConnection.ContentType, "Content type of requests sent to apiserver.")
-	fs.Float32Var(&options.config.ClientConnection.QPS, "kube-api-qps", options.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver")
-	fs.Int32Var(&options.config.ClientConnection.Burst, "kube-api-burst", options.config.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver")
-	fs.DurationVar(&options.config.UDPIdleTimeout.Duration, "udp-timeout", options.config.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace")
-	if options.config.Conntrack.Max == nil {
-		options.config.Conntrack.Max = utilpointer.Int32Ptr(0)
+	fs.StringVar(&o.config.ClientConnection.KubeConfigFile, "kubeconfig", o.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization information (the master location is set by the master flag).")
+	fs.Var(componentconfig.PortRangeVar{Val: &o.config.PortRange}, "proxy-port-range", "Range of host ports (beginPort-endPort, inclusive) that may be consumed in order to proxy service traffic. If unspecified (0-0) then ports will be randomly chosen.")
+	fs.StringVar(&o.config.HostnameOverride, "hostname-override", o.config.HostnameOverride, "If non-empty, will use this string as identification instead of the actual hostname.")
+	fs.Var(&o.config.Mode, "proxy-mode", "Which proxy mode to use: 'userspace' (older) or 'iptables' (faster) or 'ipvs' (experimental). If blank, use the best-available proxy (currently iptables).  If the iptables proxy is selected, regardless of how, but the system's kernel or iptables versions are insufficient, this always falls back to the userspace proxy.")
+	fs.Int32Var(o.config.IPTables.MasqueradeBit, "iptables-masquerade-bit", utilpointer.Int32PtrDerefOr(o.config.IPTables.MasqueradeBit, 14), "If using the pure iptables proxy, the bit of the fwmark space to mark packets requiring SNAT with.  Must be within the range [0, 31].")
+	fs.DurationVar(&o.config.IPTables.SyncPeriod.Duration, "iptables-sync-period", o.config.IPTables.SyncPeriod.Duration, "The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
+	fs.DurationVar(&o.config.IPTables.MinSyncPeriod.Duration, "iptables-min-sync-period", o.config.IPTables.MinSyncPeriod.Duration, "The minimum interval of how often the iptables rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
+	fs.DurationVar(&o.config.IPVS.SyncPeriod.Duration, "ipvs-sync-period", o.config.IPVS.SyncPeriod.Duration, "The maximum interval of how often ipvs rules are refreshed (e.g. '5s', '1m', '2h22m').  Must be greater than 0.")
+	fs.DurationVar(&o.config.IPVS.MinSyncPeriod.Duration, "ipvs-min-sync-period", o.config.IPVS.MinSyncPeriod.Duration, "The minimum interval of how often the ipvs rules can be refreshed as endpoints and services change (e.g. '5s', '1m', '2h22m').")
+	fs.DurationVar(&o.config.ConfigSyncPeriod.Duration, "config-sync-period", o.config.ConfigSyncPeriod.Duration, "How often configuration from the apiserver is refreshed.  Must be greater than 0.")
+	fs.BoolVar(&o.config.IPTables.MasqueradeAll, "masquerade-all", o.config.IPTables.MasqueradeAll, "If using the pure iptables proxy, SNAT all traffic sent via Service cluster IPs (this not commonly needed)")
+	fs.StringVar(&o.config.ClusterCIDR, "cluster-cidr", o.config.ClusterCIDR, "The CIDR range of pods in the cluster. When configured, traffic sent to a Service cluster IP from outside this range will be masqueraded and traffic sent from pods to an external LoadBalancer IP will be directed to the respective cluster IP instead")
+	fs.StringVar(&o.config.ClientConnection.ContentType, "kube-api-content-type", o.config.ClientConnection.ContentType, "Content type of requests sent to apiserver.")
+	fs.Float32Var(&o.config.ClientConnection.QPS, "kube-api-qps", o.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver")
+	fs.Int32Var(&o.config.ClientConnection.Burst, "kube-api-burst", o.config.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver")
+	fs.DurationVar(&o.config.UDPIdleTimeout.Duration, "udp-timeout", o.config.UDPIdleTimeout.Duration, "How long an idle UDP connection will be kept open (e.g. '250ms', '2s').  Must be greater than 0. Only applicable for proxy-mode=userspace")
+	if o.config.Conntrack.Max == nil {
+		o.config.Conntrack.Max = utilpointer.Int32Ptr(0)
 	}
-	fs.Int32Var(options.config.Conntrack.Max, "conntrack-max", *options.config.Conntrack.Max,
+	fs.Int32Var(o.config.Conntrack.Max, "conntrack-max", *o.config.Conntrack.Max,
 		"Maximum number of NAT connections to track (0 to leave as-is). This overrides conntrack-max-per-core and conntrack-min.")
 	fs.MarkDeprecated("conntrack-max", "This feature will be removed in a later release.")
-	fs.Int32Var(options.config.Conntrack.MaxPerCore, "conntrack-max-per-core", *options.config.Conntrack.MaxPerCore,
+	fs.Int32Var(o.config.Conntrack.MaxPerCore, "conntrack-max-per-core", *o.config.Conntrack.MaxPerCore,
 		"Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min).")
-	fs.Int32Var(options.config.Conntrack.Min, "conntrack-min", *options.config.Conntrack.Min,
+	fs.Int32Var(o.config.Conntrack.Min, "conntrack-min", *o.config.Conntrack.Min,
 		"Minimum number of conntrack entries to allocate, regardless of conntrack-max-per-core (set conntrack-max-per-core=0 to leave the limit as-is).")
-	fs.DurationVar(&options.config.Conntrack.TCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", options.config.Conntrack.TCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
+	fs.DurationVar(&o.config.Conntrack.TCPEstablishedTimeout.Duration, "conntrack-tcp-timeout-established", o.config.Conntrack.TCPEstablishedTimeout.Duration, "Idle timeout for established TCP connections (0 to leave as-is)")
 	fs.DurationVar(
-		&options.config.Conntrack.TCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait",
-		options.config.Conntrack.TCPCloseWaitTimeout.Duration,
+		&o.config.Conntrack.TCPCloseWaitTimeout.Duration, "conntrack-tcp-timeout-close-wait",
+		o.config.Conntrack.TCPCloseWaitTimeout.Duration,
 		"NAT timeout for TCP connections in the CLOSE_WAIT state")
-	fs.BoolVar(&options.config.EnableProfiling, "profiling", options.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.")
-	fs.StringVar(&options.config.IPVS.Scheduler, "ipvs-scheduler", options.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs")
-	utilfeature.DefaultFeatureGate.AddFlag(fs)
+	fs.BoolVar(&o.config.EnableProfiling, "profiling", o.config.EnableProfiling, "If true enables profiling via web interface on /debug/pprof handler.")
+	fs.StringVar(&o.config.IPVS.Scheduler, "ipvs-scheduler", o.config.IPVS.Scheduler, "The ipvs scheduler type when proxy mode is ipvs")
+	fs.StringSliceVar(&o.config.NodePortAddresses, "nodeport-addresses", o.config.NodePortAddresses,
+		"A string slice of values which specify the addresses to use for NodePorts. Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32). The default empty string slice ([]) means to use all local addresses.")
+	fs.Var(flag.NewMapStringBool(&o.config.FeatureGates), "feature-gates", "A set of key=value pairs that describe feature gates for alpha/experimental features. "+
+		"Options are:\n"+strings.Join(utilfeature.DefaultFeatureGate.KnownFeatures(), "\n"))
 }
 
 func NewOptions() *Options {
@@ -192,11 +198,14 @@ func (o *Options) Complete() error {
 			return err
 		} else {
 			o.config = c
-			// Make sure we apply the feature gate settings in the config file.
-			utilfeature.DefaultFeatureGate.Set(o.config.FeatureGates)
 		}
 	}
 
+	err := utilfeature.DefaultFeatureGate.SetFromMap(o.config.FeatureGates)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -333,6 +342,8 @@ addon that provides cluster DNS for these cluster IPs. The user must create a se
 with the apiserver API to configure the proxy.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
 			cmdutil.CheckErr(opts.Complete())
 			cmdutil.CheckErr(opts.Validate(args))
 			cmdutil.CheckErr(opts.Run())
@@ -345,8 +356,7 @@ with the apiserver API to configure the proxy.`,
 		glog.Fatalf("unable to create flag defaults: %v", err)
 	}
 
-	flags := cmd.Flags()
-	AddFlags(opts, flags)
+	opts.AddFlags(cmd.Flags())
 
 	cmd.MarkFlagFilename("config", "yaml", "yml", "json")
 
@@ -464,17 +474,14 @@ func (s *ProxyServer) Run() error {
 
 	// Start up a metrics server if requested
 	if len(s.MetricsBindAddress) > 0 {
-		mux := http.NewServeMux()
+		mux := mux.NewPathRecorderMux("kube-proxy")
 		healthz.InstallHandler(mux)
 		mux.HandleFunc("/proxyMode", func(w http.ResponseWriter, r *http.Request) {
 			fmt.Fprintf(w, "%s", s.ProxyMode)
 		})
 		mux.Handle("/metrics", prometheus.Handler())
 		if s.EnableProfiling {
-			mux.HandleFunc("/debug/pprof/", pprof.Index)
-			mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
-			mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
-			mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+			routes.Profiling{}.Install(mux)
 		}
 		configz.InstallHandler(mux)
 		go wait.Until(func() {

vendor/k8s.io/kubernetes/cmd/kube-proxy/app/server_others.go

@@ -83,6 +83,7 @@ func newProxyServer(
 
 	var iptInterface utiliptables.Interface
 	var ipvsInterface utilipvs.Interface
+	var kernelHandler ipvs.KernelHandler
 	var ipsetInterface utilipset.Interface
 	var dbus utildbus.Interface
 
@@ -92,6 +93,7 @@ func newProxyServer(
 	dbus = utildbus.New()
 	iptInterface = utiliptables.New(execer, dbus, protocol)
 	ipvsInterface = utilipvs.New(execer)
+	kernelHandler = ipvs.NewLinuxKernelHandler()
 	ipsetInterface = utilipset.New(execer)
 
 	// We omit creation of pretty much everything if we run in cleanup mode
@@ -133,7 +135,7 @@ func newProxyServer(
 	var serviceEventHandler proxyconfig.ServiceHandler
 	var endpointsEventHandler proxyconfig.EndpointsHandler
 
-	proxyMode := getProxyMode(string(config.Mode), iptInterface, ipsetInterface, iptables.LinuxKernelCompatTester{})
+	proxyMode := getProxyMode(string(config.Mode), iptInterface, kernelHandler, ipsetInterface, iptables.LinuxKernelCompatTester{})
 	if proxyMode == proxyModeIPTables {
 		glog.V(0).Info("Using iptables Proxier.")
 		nodeIP := net.ParseIP(config.BindAddress)
@@ -159,6 +161,7 @@ func newProxyServer(
 			nodeIP,
 			recorder,
 			healthzUpdater,
+			config.NodePortAddresses,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -194,6 +197,7 @@ func newProxyServer(
 			recorder,
 			healthzServer,
 			config.IPVS.Scheduler,
+			config.NodePortAddresses,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -224,6 +228,7 @@ func newProxyServer(
 			config.IPTables.SyncPeriod.Duration,
 			config.IPTables.MinSyncPeriod.Duration,
 			config.UDPIdleTimeout.Duration,
+			config.NodePortAddresses,
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create proxier: %v", err)
@@ -269,7 +274,7 @@ func newProxyServer(
 	}, nil
 }
 
-func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
+func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
 	if proxyMode == proxyModeUserspace {
 		return proxyModeUserspace
 	}
@@ -280,7 +285,7 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.SupportIPVSProxyMode) {
 		if proxyMode == proxyModeIPVS {
-			return tryIPVSProxy(iptver, ipsetver, kcompat)
+			return tryIPVSProxy(iptver, khandle, ipsetver, kcompat)
 		} else {
 			glog.Warningf("Can't use ipvs proxier, trying iptables proxier")
 			return tryIPTablesProxy(iptver, kcompat)
@@ -290,10 +295,10 @@ func getProxyMode(proxyMode string, iptver iptables.IPTablesVersioner, ipsetver
 	return tryIPTablesProxy(iptver, kcompat)
 }
 
-func tryIPVSProxy(iptver iptables.IPTablesVersioner, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
+func tryIPVSProxy(iptver iptables.IPTablesVersioner, khandle ipvs.KernelHandler, ipsetver ipvs.IPSetVersioner, kcompat iptables.KernelCompatTester) string {
 	// guaranteed false on error, error only necessary for debugging
-	// IPVS Proxier relies on ipset
-	useIPVSProxy, err := ipvs.CanUseIPVSProxier(ipsetver)
+	// IPVS Proxier relies on ip_vs_* kernel modules and ipset
+	useIPVSProxy, err := ipvs.CanUseIPVSProxier(khandle, ipsetver)
 	if err != nil {
 		// Try to fallback to iptables before falling back to userspace
 		utilruntime.HandleError(fmt.Errorf("can't determine whether to use ipvs proxy, error: %v", err))

vendor/k8s.io/kubernetes/cmd/kube-proxy/app/server_others_test.go

@@ -0,0 +1,247 @@
+// +build !windows
+
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"testing"
+
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/kubernetes/pkg/proxy/ipvs"
+	"k8s.io/kubernetes/pkg/util/iptables"
+)
+
+func Test_getProxyMode(t *testing.T) {
+	var cases = []struct {
+		flag            string
+		annotationKey   string
+		annotationVal   string
+		iptablesVersion string
+		ipsetVersion    string
+		kmods           []string
+		kernelCompat    bool
+		iptablesError   error
+		ipsetError      error
+		expected        string
+	}{
+		{ // flag says userspace
+			flag:     "userspace",
+			expected: proxyModeUserspace,
+		},
+		{ // flag says iptables, error detecting version
+			flag:          "iptables",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // flag says iptables, version too low
+			flag:            "iptables",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, version ok, kernel not compatible
+			flag:            "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, version ok, kernel is compatible
+			flag:            "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // detect, error
+			flag:          "",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // detect, version too low
+			flag:            "",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // detect, version ok, kernel not compatible
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+		{ // detect, version ok, kernel is compatible
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is compatible
+			flag:            "ipvs",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // specify ipvs, feature gateway disabled, iptables version too low
+			flag:            "ipvs",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // specify ipvs, feature gateway disabled, iptables version ok, kernel is not compatible
+			flag:            "ipvs",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+	}
+	for i, c := range cases {
+		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
+		kcompater := &fakeKernelCompatTester{c.kernelCompat}
+		ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
+		khandler := &fakeKernelHandler{c.kmods}
+		r := getProxyMode(c.flag, versioner, khandler, ipsetver, kcompater)
+		if r != c.expected {
+			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
+		}
+	}
+}
+
+// This is a coarse test, but it offers some modicum of confidence as the code is evolved.
+func Test_getProxyModeEnableFeatureGateway(t *testing.T) {
+	// enable IPVS feature gateway
+	utilfeature.DefaultFeatureGate.Set("SupportIPVSProxyMode=true")
+
+	var cases = []struct {
+		flag            string
+		iptablesVersion string
+		ipsetVersion    string
+		kernelCompat    bool
+		iptablesError   error
+		ipsetError      error
+		mods            []string
+		expected        string
+	}{
+		{ // flag says userspace
+			flag:     "userspace",
+			expected: proxyModeUserspace,
+		},
+		{ // flag says iptables, error detecting version
+			flag:          "iptables",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // flag says iptables, version too low
+			flag:            "iptables",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, version ok, kernel not compatible
+			flag:            "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says iptables, version ok, kernel is compatible
+			flag:            "iptables",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // detect, error
+			flag:          "",
+			iptablesError: fmt.Errorf("oops!"),
+			expected:      proxyModeUserspace,
+		},
+		{ // detect, version too low
+			flag:            "",
+			iptablesVersion: "0.0.0",
+			expected:        proxyModeUserspace,
+		},
+		{ // detect, version ok, kernel not compatible
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+		{ // detect, version ok, kernel is compatible
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // detect, version ok, kernel is compatible
+			flag:            "",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, ipset version ok, kernel modules installed
+			flag:         "ipvs",
+			mods:         []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion: ipvs.MinIPSetCheckVersion,
+			expected:     proxyModeIPVS,
+		},
+		{ // flag says ipvs, ipset version too low, fallback on iptables mode
+			flag:            "ipvs",
+			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "0.0",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, bad ipset version, fallback on iptables mode
+			flag:            "ipvs",
+			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "a.b.c",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, required kernel modules are not installed, fallback on iptables mode
+			flag:            "ipvs",
+			mods:            []string{"foo", "bar", "baz"},
+			ipsetVersion:    ipvs.MinIPSetCheckVersion,
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    true,
+			expected:        proxyModeIPTables,
+		},
+		{ // flag says ipvs, required kernel modules are not installed, iptables version too old, fallback on userspace mode
+			flag:            "ipvs",
+			mods:            []string{"foo", "bar", "baz"},
+			ipsetVersion:    ipvs.MinIPSetCheckVersion,
+			iptablesVersion: "0.0.0",
+			kernelCompat:    true,
+			expected:        proxyModeUserspace,
+		},
+		{ // flag says ipvs, ipset version too low, iptables version too old, kernel not compatible, fallback on userspace mode
+			flag:            "ipvs",
+			mods:            []string{"ip_vs", "ip_vs_rr", "ip_vs_wrr", "ip_vs_sh", "nf_conntrack_ipv4"},
+			ipsetVersion:    "0.0",
+			iptablesVersion: iptables.MinCheckVersion,
+			kernelCompat:    false,
+			expected:        proxyModeUserspace,
+		},
+	}
+	for i, c := range cases {
+		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
+		kcompater := &fakeKernelCompatTester{c.kernelCompat}
+		ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
+		khandle := &fakeKernelHandler{c.mods}
+		r := getProxyMode(c.flag, versioner, khandle, ipsetver, kcompater)
+		if r != c.expected {
+			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
+		}
+	}
+}

vendor/k8s.io/kubernetes/cmd/kube-proxy/app/server_test.go

@@ -29,9 +29,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
 	api "k8s.io/kubernetes/pkg/apis/core"
+	"k8s.io/kubernetes/pkg/features"
 	"k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig"
 	"k8s.io/kubernetes/pkg/util/configz"
-	"k8s.io/kubernetes/pkg/util/iptables"
 	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
 
@@ -52,6 +52,10 @@ func (fake *fakeIPTablesVersioner) GetVersion() (string, error) {
 	return fake.version, fake.err
 }
 
+func (fake *fakeIPTablesVersioner) IsCompatible() error {
+	return fake.err
+}
+
 type fakeIPSetVersioner struct {
 	version string // what to return
 	err     error  // what to return
@@ -72,79 +76,13 @@ func (fake *fakeKernelCompatTester) IsCompatible() error {
 	return nil
 }
 
-func Test_getProxyMode(t *testing.T) {
-	if runtime.GOOS != "linux" {
-		t.Skip("skipping on non-Linux")
-	}
-	var cases = []struct {
-		flag            string
-		annotationKey   string
-		annotationVal   string
-		iptablesVersion string
-		ipsetVersion    string
-		kernelCompat    bool
-		iptablesError   error
-		ipsetError      error
-		expected        string
-	}{
-		{ // flag says userspace
-			flag:     "userspace",
-			expected: proxyModeUserspace,
-		},
-		{ // flag says iptables, error detecting version
-			flag:          "iptables",
-			iptablesError: fmt.Errorf("oops!"),
-			expected:      proxyModeUserspace,
-		},
-		{ // flag says iptables, version too low
-			flag:            "iptables",
-			iptablesVersion: "0.0.0",
-			expected:        proxyModeUserspace,
-		},
-		{ // flag says iptables, version ok, kernel not compatible
-			flag:            "iptables",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    false,
-			expected:        proxyModeUserspace,
-		},
-		{ // flag says iptables, version ok, kernel is compatible
-			flag:            "iptables",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-		{ // detect, error
-			flag:          "",
-			iptablesError: fmt.Errorf("oops!"),
-			expected:      proxyModeUserspace,
-		},
-		{ // detect, version too low
-			flag:            "",
-			iptablesVersion: "0.0.0",
-			expected:        proxyModeUserspace,
-		},
-		{ // detect, version ok, kernel not compatible
-			flag:            "",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    false,
-			expected:        proxyModeUserspace,
-		},
-		{ // detect, version ok, kernel is compatible
-			flag:            "",
-			iptablesVersion: iptables.MinCheckVersion,
-			kernelCompat:    true,
-			expected:        proxyModeIPTables,
-		},
-	}
-	for i, c := range cases {
-		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
-		kcompater := &fakeKernelCompatTester{c.kernelCompat}
-		ipsetver := &fakeIPSetVersioner{c.ipsetVersion, c.ipsetError}
-		r := getProxyMode(c.flag, versioner, ipsetver, kcompater)
-		if r != c.expected {
-			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
-		}
-	}
+// fakeKernelHandler implements KernelHandler.
+type fakeKernelHandler struct {
+	modules []string
+}
+
+func (fake *fakeKernelHandler) GetModules() ([]string, error) {
+	return fake.modules, nil
 }
 
 // This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
@@ -251,7 +189,8 @@ conntrack:
   min: 1
   tcpCloseWaitTimeout: 10s
   tcpEstablishedTimeout: 20s
-featureGates: "all"
+featureGates:
+  SupportIPVSProxyMode: true
 healthzBindAddress: "%s"
 hostnameOverride: "foo"
 iptables:
@@ -268,7 +207,10 @@ mode: "%s"
 oomScoreAdj: 17
 portRange: "2-7"
 resourceContainer: /foo
-udpTimeoutMilliseconds: 123ms
+udpIdleTimeout: 123ms
+nodePortAddresses:
+  - "10.20.30.40/16"
+  - "fd00:1::0/64"
 `
 
 	testCases := []struct {
@@ -362,7 +304,7 @@ udpTimeoutMilliseconds: 123ms
 				TCPCloseWaitTimeout:   &metav1.Duration{Duration: 10 * time.Second},
 				TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second},
 			},
-			FeatureGates:       "all",
+			FeatureGates:       map[string]bool{string(features.SupportIPVSProxyMode): true},
 			HealthzBindAddress: tc.healthzBindAddress,
 			HostnameOverride:   "foo",
 			IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{
@@ -381,6 +323,7 @@ udpTimeoutMilliseconds: 123ms
 			PortRange:          "2-7",
 			ResourceContainer:  "/foo",
 			UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
+			NodePortAddresses:  []string{"10.20.30.40/16", "fd00:1::0/64"},
 		}
 
 		options := NewOptions()

vendor/k8s.io/kubernetes/cmd/kube-proxy/proxy.go

@@ -19,7 +19,9 @@ package main
 import (
 	goflag "flag"
 	"fmt"
+	"math/rand"
 	"os"
+	"time"
 
 	"github.com/spf13/pflag"
 
@@ -31,6 +33,8 @@ import (
 )
 
 func main() {
+	rand.Seed(time.Now().UTC().UnixNano())
+
 	command := app.NewProxyCommand()
 
 	// TODO: once we switch everything over to Cobra commands, we can go back to calling

vendor/k8s.io/kubernetes/cmd/kube-scheduler/BUILD

@@ -5,17 +5,26 @@ load(
     "go_binary",
     "go_library",
 )
+load("//pkg/version:def.bzl", "version_x_defs")
+
+go_binary(
+    name = "kube-scheduler",
+    embed = [":go_default_library"],
+    pure = "on",
+    x_defs = version_x_defs(),
+)
 
 go_library(
     name = "go_default_library",
-    srcs = ["main.go"],
-    importpath = "k8s.io/kubernetes/cmd/gke-certificates-controller",
+    srcs = ["scheduler.go"],
+    importpath = "k8s.io/kubernetes/cmd/kube-scheduler",
     deps = [
-        "//cmd/gke-certificates-controller/app:go_default_library",
-        "//pkg/kubectl/util/logs:go_default_library",
-        "//pkg/version/verflag:go_default_library",
+        "//cmd/kube-scheduler/app:go_default_library",
+        "//pkg/client/metrics/prometheus:go_default_library",
+        "//pkg/version/prometheus:go_default_library",
         "//vendor/github.com/spf13/pflag:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
     ],
 )
 
@@ -30,13 +39,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
-        "//cmd/gke-certificates-controller/app:all-srcs",
+        "//cmd/kube-scheduler/app:all-srcs",
     ],
     tags = ["automanaged"],
 )
-
-go_binary(
-    name = "gke-certificates-controller",
-    importpath = "k8s.io/kubernetes/cmd/gke-certificates-controller",
-    library = ":go_default_library",
-)

vendor/k8s.io/kubernetes/cmd/kube-scheduler/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+- sig-scheduling-maintainers
+reviewers:
+- sig-scheduling

vendor/k8s.io/kubernetes/cmd/kube-scheduler/app/BUILD

@@ -0,0 +1,70 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["server.go"],
+    importpath = "k8s.io/kubernetes/cmd/kube-scheduler/app",
+    deps = [
+        "//pkg/api/legacyscheme:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/componentconfig/v1alpha1:go_default_library",
+        "//pkg/client/leaderelectionconfig:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/features:go_default_library",
+        "//pkg/kubectl/cmd/util:go_default_library",
+        "//pkg/master/ports:go_default_library",
+        "//pkg/scheduler:go_default_library",
+        "//pkg/scheduler/algorithmprovider:go_default_library",
+        "//pkg/scheduler/api:go_default_library",
+        "//pkg/scheduler/api/latest:go_default_library",
+        "//pkg/scheduler/factory:go_default_library",
+        "//pkg/util/configz:go_default_library",
+        "//pkg/util/flag:go_default_library",
+        "//pkg/version:go_default_library",
+        "//pkg/version/verflag:go_default_library",
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+        "//vendor/github.com/spf13/cobra:go_default_library",
+        "//vendor/github.com/spf13/pflag:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/runtime/serializer:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/mux:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/routes:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
+        "//vendor/k8s.io/client-go/informers:go_default_library",
+        "//vendor/k8s.io/client-go/informers/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/informers/storage/v1:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes:go_default_library",
+        "//vendor/k8s.io/client-go/kubernetes/typed/core/v1:go_default_library",
+        "//vendor/k8s.io/client-go/rest:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd:go_default_library",
+        "//vendor/k8s.io/client-go/tools/clientcmd/api:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection:go_default_library",
+        "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
+        "//vendor/k8s.io/client-go/tools/record:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

vendor/k8s.io/kubernetes/cmd/kube-scheduler/app/server.go

@@ -0,0 +1,709 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package app implements a Server object for running the scheduler.
+package app
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"reflect"
+	goruntime "runtime"
+	"strconv"
+	"time"
+
+	"k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/mux"
+	"k8s.io/apiserver/pkg/server/routes"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/client-go/informers"
+	coreinformers "k8s.io/client-go/informers/core/v1"
+	storageinformers "k8s.io/client-go/informers/storage/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/tools/record"
+	"k8s.io/kubernetes/pkg/api/legacyscheme"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	componentconfigv1alpha1 "k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
+	"k8s.io/kubernetes/pkg/client/leaderelectionconfig"
+	"k8s.io/kubernetes/pkg/controller"
+	"k8s.io/kubernetes/pkg/features"
+	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/pkg/scheduler"
+	"k8s.io/kubernetes/pkg/scheduler/algorithmprovider"
+	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
+	latestschedulerapi "k8s.io/kubernetes/pkg/scheduler/api/latest"
+	"k8s.io/kubernetes/pkg/scheduler/factory"
+	"k8s.io/kubernetes/pkg/util/configz"
+	utilflag "k8s.io/kubernetes/pkg/util/flag"
+	"k8s.io/kubernetes/pkg/version"
+	"k8s.io/kubernetes/pkg/version/verflag"
+
+	"github.com/golang/glog"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+// SchedulerServer has all the context and params needed to run a Scheduler
+type Options struct {
+	// ConfigFile is the location of the scheduler server's configuration file.
+	ConfigFile string
+
+	// config is the scheduler server's configuration object.
+	config *componentconfig.KubeSchedulerConfiguration
+
+	scheme *runtime.Scheme
+	codecs serializer.CodecFactory
+
+	// The fields below here are placeholders for flags that can't be directly
+	// mapped into componentconfig.KubeSchedulerConfiguration.
+	//
+	// TODO remove these fields once the deprecated flags are removed.
+
+	// master is the address of the Kubernetes API server (overrides any
+	// value in kubeconfig).
+	master                   string
+	healthzAddress           string
+	healthzPort              int32
+	policyConfigFile         string
+	policyConfigMapName      string
+	policyConfigMapNamespace string
+	useLegacyPolicyConfig    bool
+	algorithmProvider        string
+}
+
+// AddFlags adds flags for a specific SchedulerServer to the specified FlagSet
+func (o *Options) AddFlags(fs *pflag.FlagSet) {
+	fs.StringVar(&o.ConfigFile, "config", o.ConfigFile, "The path to the configuration file.")
+
+	// All flags below here are deprecated and will eventually be removed.
+
+	fs.Int32Var(&o.healthzPort, "port", ports.SchedulerPort, "The port that the scheduler's http service runs on")
+	fs.StringVar(&o.healthzAddress, "address", o.healthzAddress, "The IP address to serve on (set to 0.0.0.0 for all IPv4 interfaces and :: for all IPv6 interfaces).")
+	fs.StringVar(&o.algorithmProvider, "algorithm-provider", o.algorithmProvider, "The scheduling algorithm provider to use, one of: "+factory.ListAlgorithmProviders())
+	fs.StringVar(&o.policyConfigFile, "policy-config-file", o.policyConfigFile, "File with scheduler policy configuration. This file is used if policy ConfigMap is not provided or --use-legacy-policy-config==true")
+	usage := fmt.Sprintf("Name of the ConfigMap object that contains scheduler's policy configuration. It must exist in the system namespace before scheduler initialization if --use-legacy-policy-config==false. The config must be provided as the value of an element in 'Data' map with the key='%v'", componentconfig.SchedulerPolicyConfigMapKey)
+	fs.StringVar(&o.policyConfigMapName, "policy-configmap", o.policyConfigMapName, usage)
+	fs.StringVar(&o.policyConfigMapNamespace, "policy-configmap-namespace", o.policyConfigMapNamespace, "The namespace where policy ConfigMap is located. The system namespace will be used if this is not provided or is empty.")
+	fs.BoolVar(&o.useLegacyPolicyConfig, "use-legacy-policy-config", false, "When set to true, scheduler will ignore policy ConfigMap and uses policy config file")
+	fs.BoolVar(&o.config.EnableProfiling, "profiling", o.config.EnableProfiling, "Enable profiling via web interface host:port/debug/pprof/")
+	fs.BoolVar(&o.config.EnableContentionProfiling, "contention-profiling", o.config.EnableContentionProfiling, "Enable lock contention profiling, if profiling is enabled")
+	fs.StringVar(&o.master, "master", o.master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
+	fs.StringVar(&o.config.ClientConnection.KubeConfigFile, "kubeconfig", o.config.ClientConnection.KubeConfigFile, "Path to kubeconfig file with authorization and master location information.")
+	fs.StringVar(&o.config.ClientConnection.ContentType, "kube-api-content-type", o.config.ClientConnection.ContentType, "Content type of requests sent to apiserver.")
+	fs.Float32Var(&o.config.ClientConnection.QPS, "kube-api-qps", o.config.ClientConnection.QPS, "QPS to use while talking with kubernetes apiserver")
+	fs.Int32Var(&o.config.ClientConnection.Burst, "kube-api-burst", o.config.ClientConnection.Burst, "Burst to use while talking with kubernetes apiserver")
+	fs.StringVar(&o.config.SchedulerName, "scheduler-name", o.config.SchedulerName, "Name of the scheduler, used to select which pods will be processed by this scheduler, based on pod's \"spec.SchedulerName\".")
+	fs.StringVar(&o.config.LeaderElection.LockObjectNamespace, "lock-object-namespace", o.config.LeaderElection.LockObjectNamespace, "Define the namespace of the lock object.")
+	fs.StringVar(&o.config.LeaderElection.LockObjectName, "lock-object-name", o.config.LeaderElection.LockObjectName, "Define the name of the lock object.")
+	fs.Int32Var(&o.config.HardPodAffinitySymmetricWeight, "hard-pod-affinity-symmetric-weight", o.config.HardPodAffinitySymmetricWeight,
+		"RequiredDuringScheduling affinity is not symmetric, but there is an implicit PreferredDuringScheduling affinity rule corresponding "+
+			"to every RequiredDuringScheduling affinity rule. --hard-pod-affinity-symmetric-weight represents the weight of implicit PreferredDuringScheduling affinity rule.")
+	fs.MarkDeprecated("hard-pod-affinity-symmetric-weight", "This option was moved to the policy configuration file")
+	fs.StringVar(&o.config.FailureDomains, "failure-domains", o.config.FailureDomains, "Indicate the \"all topologies\" set for an empty topologyKey when it's used for PreferredDuringScheduling pod anti-affinity.")
+	fs.MarkDeprecated("failure-domains", "Doesn't have any effect. Will be removed in future version.")
+	leaderelectionconfig.BindFlags(&o.config.LeaderElection.LeaderElectionConfiguration, fs)
+	utilfeature.DefaultFeatureGate.AddFlag(fs)
+}
+
+func NewOptions() (*Options, error) {
+	o := &Options{
+		config: new(componentconfig.KubeSchedulerConfiguration),
+	}
+
+	o.scheme = runtime.NewScheme()
+	o.codecs = serializer.NewCodecFactory(o.scheme)
+
+	if err := componentconfig.AddToScheme(o.scheme); err != nil {
+		return nil, err
+	}
+	if err := componentconfigv1alpha1.AddToScheme(o.scheme); err != nil {
+		return nil, err
+	}
+
+	return o, nil
+}
+
+func (o *Options) Complete() error {
+	if len(o.ConfigFile) == 0 {
+		glog.Warning("WARNING: all flags other than --config are deprecated. Please begin using a config file ASAP.")
+		o.applyDeprecatedHealthzAddressToConfig()
+		o.applyDeprecatedHealthzPortToConfig()
+		o.applyDeprecatedAlgorithmSourceOptionsToConfig()
+	}
+
+	return nil
+}
+
+// applyDeprecatedHealthzAddressToConfig sets o.config.HealthzBindAddress and
+// o.config.MetricsBindAddress from flags passed on the command line based on
+// the following rules:
+//
+// 1. If --address is empty, leave the config as-is.
+// 2. Otherwise, use the value of --address for the address portion of
+//    o.config.HealthzBindAddress
+func (o *Options) applyDeprecatedHealthzAddressToConfig() {
+	if len(o.healthzAddress) == 0 {
+		return
+	}
+
+	_, port, err := net.SplitHostPort(o.config.HealthzBindAddress)
+	if err != nil {
+		glog.Fatalf("invalid healthz bind address %q: %v", o.config.HealthzBindAddress, err)
+	}
+	o.config.HealthzBindAddress = net.JoinHostPort(o.healthzAddress, port)
+	o.config.MetricsBindAddress = net.JoinHostPort(o.healthzAddress, port)
+}
+
+// applyDeprecatedHealthzPortToConfig sets o.config.HealthzBindAddress and
+// o.config.MetricsBindAddress from flags passed on the command line based on
+// the following rules:
+//
+// 1. If --port is -1, disable the healthz server.
+// 2. Otherwise, use the value of --port for the port portion of
+//    o.config.HealthzBindAddress
+func (o *Options) applyDeprecatedHealthzPortToConfig() {
+	if o.healthzPort == -1 {
+		o.config.HealthzBindAddress = ""
+		o.config.MetricsBindAddress = ""
+		return
+	}
+
+	host, _, err := net.SplitHostPort(o.config.HealthzBindAddress)
+	if err != nil {
+		glog.Fatalf("invalid healthz bind address %q: %v", o.config.HealthzBindAddress, err)
+	}
+	o.config.HealthzBindAddress = net.JoinHostPort(host, strconv.Itoa(int(o.healthzPort)))
+	o.config.MetricsBindAddress = net.JoinHostPort(host, strconv.Itoa(int(o.healthzPort)))
+}
+
+// applyDeprecatedAlgorithmSourceOptionsToConfig sets o.config.AlgorithmSource from
+// flags passed on the command line in the following precedence order:
+//
+// 1. --use-legacy-policy-config to use a policy file.
+// 2. --policy-configmap to use a policy config map value.
+// 3. --algorithm-provider to use a named algorithm provider.
+func (o *Options) applyDeprecatedAlgorithmSourceOptionsToConfig() {
+	switch {
+	case o.useLegacyPolicyConfig || (len(o.policyConfigFile) > 0 && o.policyConfigMapName == ""):
+		o.config.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Policy: &componentconfig.SchedulerPolicySource{
+				File: &componentconfig.SchedulerPolicyFileSource{
+					Path: o.policyConfigFile,
+				},
+			},
+		}
+	case len(o.policyConfigMapName) > 0:
+		o.config.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Policy: &componentconfig.SchedulerPolicySource{
+				ConfigMap: &componentconfig.SchedulerPolicyConfigMapSource{
+					Name:      o.policyConfigMapName,
+					Namespace: o.policyConfigMapNamespace,
+				},
+			},
+		}
+	case len(o.algorithmProvider) > 0:
+		o.config.AlgorithmSource = componentconfig.SchedulerAlgorithmSource{
+			Provider: &o.algorithmProvider,
+		}
+	}
+}
+
+// Validate validates all the required options.
+func (o *Options) Validate(args []string) error {
+	if len(args) != 0 {
+		return errors.New("no arguments are supported")
+	}
+
+	return nil
+}
+
+// loadConfigFromFile loads the contents of file and decodes it as a
+// KubeSchedulerConfiguration object.
+func (o *Options) loadConfigFromFile(file string) (*componentconfig.KubeSchedulerConfiguration, error) {
+	data, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+
+	return o.loadConfig(data)
+}
+
+// loadConfig decodes data as a KubeSchedulerConfiguration object.
+func (o *Options) loadConfig(data []byte) (*componentconfig.KubeSchedulerConfiguration, error) {
+	configObj, gvk, err := o.codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	config, ok := configObj.(*componentconfig.KubeSchedulerConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+	return config, nil
+}
+
+func (o *Options) ApplyDefaults(in *componentconfig.KubeSchedulerConfiguration) (*componentconfig.KubeSchedulerConfiguration, error) {
+	external, err := o.scheme.ConvertToVersion(in, componentconfigv1alpha1.SchemeGroupVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	o.scheme.Default(external)
+
+	internal, err := o.scheme.ConvertToVersion(external, componentconfig.SchemeGroupVersion)
+	if err != nil {
+		return nil, err
+	}
+
+	out := internal.(*componentconfig.KubeSchedulerConfiguration)
+
+	return out, nil
+}
+
+func (o *Options) Run() error {
+	config := o.config
+
+	if len(o.ConfigFile) > 0 {
+		if c, err := o.loadConfigFromFile(o.ConfigFile); err != nil {
+			return err
+		} else {
+			config = c
+		}
+	}
+
+	// Apply algorithms based on feature gates.
+	// TODO: make configurable?
+	algorithmprovider.ApplyFeatureGates()
+
+	server, err := NewSchedulerServer(config, o.master)
+	if err != nil {
+		return err
+	}
+
+	stop := make(chan struct{})
+	return server.Run(stop)
+}
+
+// NewSchedulerCommand creates a *cobra.Command object with default parameters
+func NewSchedulerCommand() *cobra.Command {
+	opts, err := NewOptions()
+	if err != nil {
+		glog.Fatalf("unable to initialize command options: %v", err)
+	}
+
+	cmd := &cobra.Command{
+		Use: "kube-scheduler",
+		Long: `The Kubernetes scheduler is a policy-rich, topology-aware,
+workload-specific function that significantly impacts availability, performance,
+and capacity. The scheduler needs to take into account individual and collective
+resource requirements, quality of service requirements, hardware/software/policy
+constraints, affinity and anti-affinity specifications, data locality, inter-workload
+interference, deadlines, and so on. Workload-specific requirements will be exposed
+through the API as necessary.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			verflag.PrintAndExitIfRequested()
+			utilflag.PrintFlags(cmd.Flags())
+
+			cmdutil.CheckErr(opts.Complete())
+			cmdutil.CheckErr(opts.Validate(args))
+			cmdutil.CheckErr(opts.Run())
+		},
+	}
+
+	opts.config, err = opts.ApplyDefaults(opts.config)
+	if err != nil {
+		glog.Fatalf("unable to apply config defaults: %v", err)
+	}
+
+	opts.AddFlags(cmd.Flags())
+	cmd.MarkFlagFilename("config", "yaml", "yml", "json")
+
+	return cmd
+}
+
+// SchedulerServer represents all the parameters required to start the
+// Kubernetes scheduler server.
+type SchedulerServer struct {
+	SchedulerName                  string
+	Client                         clientset.Interface
+	InformerFactory                informers.SharedInformerFactory
+	PodInformer                    coreinformers.PodInformer
+	AlgorithmSource                componentconfig.SchedulerAlgorithmSource
+	HardPodAffinitySymmetricWeight int32
+	EventClient                    v1core.EventsGetter
+	Recorder                       record.EventRecorder
+	Broadcaster                    record.EventBroadcaster
+	// LeaderElection is optional.
+	LeaderElection *leaderelection.LeaderElectionConfig
+	// HealthzServer is optional.
+	HealthzServer *http.Server
+	// MetricsServer is optional.
+	MetricsServer *http.Server
+}
+
+// NewSchedulerServer creates a runnable SchedulerServer from configuration.
+func NewSchedulerServer(config *componentconfig.KubeSchedulerConfiguration, master string) (*SchedulerServer, error) {
+	if config == nil {
+		return nil, errors.New("config is required")
+	}
+
+	// Configz registration.
+	// only register if we're actually exposing it somewhere
+	if len(config.MetricsBindAddress) > 0 || len(config.HealthzBindAddress) > 0 {
+		if c, err := configz.New("componentconfig"); err == nil {
+			c.Set(config)
+		} else {
+			return nil, fmt.Errorf("unable to register configz: %s", err)
+		}
+	}
+
+	// Prepare some Kube clients.
+	client, leaderElectionClient, eventClient, err := createClients(config.ClientConnection, master)
+	if err != nil {
+		return nil, err
+	}
+
+	// Prepare event clients.
+	eventBroadcaster := record.NewBroadcaster()
+	recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, v1.EventSource{Component: config.SchedulerName})
+
+	// Set up leader election if enabled.
+	var leaderElectionConfig *leaderelection.LeaderElectionConfig
+	if config.LeaderElection.LeaderElect {
+		leaderElectionConfig, err = makeLeaderElectionConfig(config.LeaderElection, leaderElectionClient, recorder)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Prepare a healthz server. If the metrics bind address is the same as the
+	// healthz bind address, consolidate the servers into one.
+	var healthzServer *http.Server
+	if len(config.HealthzBindAddress) > 0 {
+		healthzServer = makeHealthzServer(config)
+	}
+
+	// Prepare a separate metrics server only if the bind address differs from the
+	// healthz bind address.
+	var metricsServer *http.Server
+	if len(config.MetricsBindAddress) > 0 && config.HealthzBindAddress != config.MetricsBindAddress {
+		metricsServer = makeMetricsServer(config)
+	}
+
+	return &SchedulerServer{
+		SchedulerName:                  config.SchedulerName,
+		Client:                         client,
+		InformerFactory:                informers.NewSharedInformerFactory(client, 0),
+		PodInformer:                    factory.NewPodInformer(client, 0, config.SchedulerName),
+		AlgorithmSource:                config.AlgorithmSource,
+		HardPodAffinitySymmetricWeight: config.HardPodAffinitySymmetricWeight,
+		EventClient:                    eventClient,
+		Recorder:                       recorder,
+		Broadcaster:                    eventBroadcaster,
+		LeaderElection:                 leaderElectionConfig,
+		HealthzServer:                  healthzServer,
+		MetricsServer:                  metricsServer,
+	}, nil
+}
+
+// makeLeaderElectionConfig builds a leader election configuration. It will
+// create a new resource lock associated with the configuration.
+func makeLeaderElectionConfig(config componentconfig.KubeSchedulerLeaderElectionConfiguration, client clientset.Interface, recorder record.EventRecorder) (*leaderelection.LeaderElectionConfig, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return nil, fmt.Errorf("unable to get hostname: %v", err)
+	}
+	// add a uniquifier so that two processes on the same host don't accidentally both become active
+	id := hostname + "_" + string(uuid.NewUUID())
+
+	rl, err := resourcelock.New(config.ResourceLock,
+		config.LockObjectNamespace,
+		config.LockObjectName,
+		client.CoreV1(),
+		resourcelock.ResourceLockConfig{
+			Identity:      id,
+			EventRecorder: recorder,
+		})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create resource lock: %v", err)
+	}
+
+	return &leaderelection.LeaderElectionConfig{
+		Lock:          rl,
+		LeaseDuration: config.LeaseDuration.Duration,
+		RenewDeadline: config.RenewDeadline.Duration,
+		RetryPeriod:   config.RetryPeriod.Duration,
+	}, nil
+}
+
+// makeHealthzServer creates a healthz server from the config, and will also
+// embed the metrics handler if the healthz and metrics address configurations
+// are the same.
+func makeHealthzServer(config *componentconfig.KubeSchedulerConfiguration) *http.Server {
+	mux := mux.NewPathRecorderMux("kube-scheduler")
+	healthz.InstallHandler(mux)
+	if config.HealthzBindAddress == config.MetricsBindAddress {
+		configz.InstallHandler(mux)
+		mux.Handle("/metrics", prometheus.Handler())
+	}
+	if config.EnableProfiling {
+		routes.Profiling{}.Install(mux)
+		if config.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+	return &http.Server{
+		Addr:    config.HealthzBindAddress,
+		Handler: mux,
+	}
+}
+
+// makeMetricsServer builds a metrics server from the config.
+func makeMetricsServer(config *componentconfig.KubeSchedulerConfiguration) *http.Server {
+	mux := mux.NewPathRecorderMux("kube-scheduler")
+	configz.InstallHandler(mux)
+	mux.Handle("/metrics", prometheus.Handler())
+	if config.EnableProfiling {
+		routes.Profiling{}.Install(mux)
+		if config.EnableContentionProfiling {
+			goruntime.SetBlockProfileRate(1)
+		}
+	}
+	return &http.Server{
+		Addr:    config.MetricsBindAddress,
+		Handler: mux,
+	}
+}
+
+// createClients creates a kube client and an event client from the given config and masterOverride.
+// TODO remove masterOverride when CLI flags are removed.
+func createClients(config componentconfig.ClientConnectionConfiguration, masterOverride string) (clientset.Interface, clientset.Interface, v1core.EventsGetter, error) {
+	if len(config.KubeConfigFile) == 0 && len(masterOverride) == 0 {
+		glog.Warningf("Neither --kubeconfig nor --master was specified. Using default API client. This might not work.")
+	}
+
+	// This creates a client, first loading any specified kubeconfig
+	// file, and then overriding the Master flag, if non-empty.
+	kubeConfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: config.KubeConfigFile},
+		&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: masterOverride}}).ClientConfig()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	kubeConfig.AcceptContentTypes = config.AcceptContentTypes
+	kubeConfig.ContentType = config.ContentType
+	kubeConfig.QPS = config.QPS
+	//TODO make config struct use int instead of int32?
+	kubeConfig.Burst = int(config.Burst)
+
+	client, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "scheduler"))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeConfig, "leader-election"))
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	eventClient, err := clientset.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return client, leaderElectionClient, eventClient.CoreV1(), nil
+}
+
+// Run runs the SchedulerServer. This should never exit.
+func (s *SchedulerServer) Run(stop chan struct{}) error {
+	// To help debugging, immediately log version
+	glog.Infof("Version: %+v", version.Get())
+
+	// Build a scheduler config from the provided algorithm source.
+	schedulerConfig, err := s.SchedulerConfig()
+	if err != nil {
+		return err
+	}
+
+	// Create the scheduler.
+	sched := scheduler.NewFromConfig(schedulerConfig)
+
+	// Prepare the event broadcaster.
+	if !reflect.ValueOf(s.Broadcaster).IsNil() && !reflect.ValueOf(s.EventClient).IsNil() {
+		s.Broadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: s.EventClient.Events("")})
+	}
+
+	// Start up the healthz server.
+	if s.HealthzServer != nil {
+		go wait.Until(func() {
+			glog.Infof("starting healthz server on %v", s.HealthzServer.Addr)
+			err := s.HealthzServer.ListenAndServe()
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("failed to start healthz server: %v", err))
+			}
+		}, 5*time.Second, stop)
+	}
+
+	// Start up the metrics server.
+	if s.MetricsServer != nil {
+		go wait.Until(func() {
+			glog.Infof("starting metrics server on %v", s.MetricsServer.Addr)
+			err := s.MetricsServer.ListenAndServe()
+			if err != nil {
+				utilruntime.HandleError(fmt.Errorf("failed to start metrics server: %v", err))
+			}
+		}, 5*time.Second, stop)
+	}
+
+	// Start all informers.
+	go s.PodInformer.Informer().Run(stop)
+	s.InformerFactory.Start(stop)
+
+	// Wait for all caches to sync before scheduling.
+	s.InformerFactory.WaitForCacheSync(stop)
+	controller.WaitForCacheSync("scheduler", stop, s.PodInformer.Informer().HasSynced)
+
+	// Prepare a reusable run function.
+	run := func(stopCh <-chan struct{}) {
+		sched.Run()
+		<-stopCh
+	}
+
+	// If leader election is enabled, run via LeaderElector until done and exit.
+	if s.LeaderElection != nil {
+		s.LeaderElection.Callbacks = leaderelection.LeaderCallbacks{
+			OnStartedLeading: run,
+			OnStoppedLeading: func() {
+				utilruntime.HandleError(fmt.Errorf("lost master"))
+			},
+		}
+		leaderElector, err := leaderelection.NewLeaderElector(*s.LeaderElection)
+		if err != nil {
+			return fmt.Errorf("couldn't create leader elector: %v", err)
+		}
+
+		leaderElector.Run()
+
+		return fmt.Errorf("lost lease")
+	}
+
+	// Leader election is disabled, so run inline until done.
+	run(stop)
+	return fmt.Errorf("finished without leader elect")
+}
+
+// SchedulerConfig creates the scheduler configuration. This is exposed for use
+// by tests.
+func (s *SchedulerServer) SchedulerConfig() (*scheduler.Config, error) {
+	var storageClassInformer storageinformers.StorageClassInformer
+	if utilfeature.DefaultFeatureGate.Enabled(features.VolumeScheduling) {
+		storageClassInformer = s.InformerFactory.Storage().V1().StorageClasses()
+	}
+
+	// Set up the configurator which can create schedulers from configs.
+	configurator := factory.NewConfigFactory(
+		s.SchedulerName,
+		s.Client,
+		s.InformerFactory.Core().V1().Nodes(),
+		s.PodInformer,
+		s.InformerFactory.Core().V1().PersistentVolumes(),
+		s.InformerFactory.Core().V1().PersistentVolumeClaims(),
+		s.InformerFactory.Core().V1().ReplicationControllers(),
+		s.InformerFactory.Extensions().V1beta1().ReplicaSets(),
+		s.InformerFactory.Apps().V1beta1().StatefulSets(),
+		s.InformerFactory.Core().V1().Services(),
+		s.InformerFactory.Policy().V1beta1().PodDisruptionBudgets(),
+		storageClassInformer,
+		s.HardPodAffinitySymmetricWeight,
+		utilfeature.DefaultFeatureGate.Enabled(features.EnableEquivalenceClassCache),
+	)
+
+	source := s.AlgorithmSource
+	var config *scheduler.Config
+	switch {
+	case source.Provider != nil:
+		// Create the config from a named algorithm provider.
+		sc, err := configurator.CreateFromProvider(*source.Provider)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't create scheduler using provider %q: %v", *source.Provider, err)
+		}
+		config = sc
+	case source.Policy != nil:
+		// Create the config from a user specified policy source.
+		policy := &schedulerapi.Policy{}
+		switch {
+		case source.Policy.File != nil:
+			// Use a policy serialized in a file.
+			policyFile := source.Policy.File.Path
+			_, err := os.Stat(policyFile)
+			if err != nil {
+				return nil, fmt.Errorf("missing policy config file %s", policyFile)
+			}
+			data, err := ioutil.ReadFile(policyFile)
+			if err != nil {
+				return nil, fmt.Errorf("couldn't read policy config: %v", err)
+			}
+			err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy)
+			if err != nil {
+				return nil, fmt.Errorf("invalid policy: %v", err)
+			}
+		case source.Policy.ConfigMap != nil:
+			// Use a policy serialized in a config map value.
+			policyRef := source.Policy.ConfigMap
+			policyConfigMap, err := s.Client.CoreV1().ConfigMaps(policyRef.Namespace).Get(policyRef.Name, metav1.GetOptions{})
+			if err != nil {
+				return nil, fmt.Errorf("couldn't get policy config map %s/%s: %v", policyRef.Namespace, policyRef.Name, err)
+			}
+			data, found := policyConfigMap.Data[componentconfig.SchedulerPolicyConfigMapKey]
+			if !found {
+				return nil, fmt.Errorf("missing policy config map value at key %q", componentconfig.SchedulerPolicyConfigMapKey)
+			}
+			err = runtime.DecodeInto(latestschedulerapi.Codec, []byte(data), policy)
+			if err != nil {
+				return nil, fmt.Errorf("invalid policy: %v", err)
+			}
+		}
+		sc, err := configurator.CreateFromConfig(*policy)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't create scheduler from policy: %v", err)
+		}
+		config = sc
+	default:
+		return nil, fmt.Errorf("unsupported algorithm source: %v", source)
+	}
+	// Additional tweaks to the config produced by the configurator.
+	config.Recorder = s.Recorder
+	return config, nil
+}

vendor/k8s.io/kubernetes/cmd/kube-scheduler/scheduler.go

@@ -0,0 +1,52 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	goflag "flag"
+	"fmt"
+	"math/rand"
+	"os"
+	"time"
+
+	"github.com/spf13/pflag"
+	utilflag "k8s.io/apiserver/pkg/util/flag"
+	"k8s.io/apiserver/pkg/util/logs"
+	"k8s.io/kubernetes/cmd/kube-scheduler/app"
+	_ "k8s.io/kubernetes/pkg/client/metrics/prometheus" // for client metric registration
+	_ "k8s.io/kubernetes/pkg/version/prometheus"        // for version metric registration
+)
+
+func main() {
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	command := app.NewSchedulerCommand()
+
+	// TODO: once we switch everything over to Cobra commands, we can go back to calling
+	// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
+	// normalize func and add the go flag set by hand.
+	pflag.CommandLine.SetNormalizeFunc(utilflag.WordSepNormalizeFunc)
+	pflag.CommandLine.AddGoFlagSet(goflag.CommandLine)
+	// utilflag.InitFlags()
+	logs.InitLogs()
+	defer logs.FlushLogs()
+
+	if err := command.Execute(); err != nil {
+		fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}

vendor/k8s.io/kubernetes/cmd/kubeadm/BUILD

@@ -9,14 +9,8 @@ load("//pkg/version:def.bzl", "version_x_defs")
 
 go_binary(
     name = "kubeadm",
-    gc_linkopts = [
-        "-linkmode",
-        "external",
-        "-extldflags",
-        "-static",
-    ],
-    importpath = "k8s.io/kubernetes/cmd/kubeadm",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
+    pure = "on",
     x_defs = version_x_defs(),
 )
 

vendor/k8s.io/kubernetes/cmd/kubeadm/OWNERS

@@ -1,11 +1,9 @@
 approvers:
 - jbeda
 - luxas
-- mikedanese
 - krousey
 - timothysc
 reviewers:
-- mikedanese
 - luxas
 - dmmcquay
 - krousey
@@ -15,3 +13,4 @@ reviewers:
 - kad
 - xiangpengzhao
 - mattmoyer
+- kargakis

vendor/k8s.io/kubernetes/cmd/kubeadm/app/BUILD

@@ -44,7 +44,6 @@ filegroup(
         "//cmd/kubeadm/app/phases/kubelet:all-srcs",
         "//cmd/kubeadm/app/phases/markmaster:all-srcs",
         "//cmd/kubeadm/app/phases/selfhosting:all-srcs",
-        "//cmd/kubeadm/app/phases/token:all-srcs",
         "//cmd/kubeadm/app/phases/upgrade:all-srcs",
         "//cmd/kubeadm/app/phases/uploadconfig:all-srcs",
         "//cmd/kubeadm/app/preflight:all-srcs",

vendor/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/BUILD

@@ -11,13 +11,13 @@ go_library(
         "doc.go",
         "register.go",
         "types.go",
-        "well_known_labels.go",
         "zz_generated.deepcopy.go",
     ],
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm",
     deps = [
-        "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
+        "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",

vendor/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/fuzzer/BUILD

@@ -11,7 +11,7 @@ go_library(
     importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/fuzzer",
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm:go_default_library",
-        "//pkg/kubelet/apis/kubeletconfig/v1alpha1:go_default_library",
+        "//pkg/kubelet/apis/kubeletconfig/v1beta1:go_default_library",
         "//pkg/proxy/apis/kubeproxyconfig/v1alpha1:go_default_library",
         "//pkg/util/pointer:go_default_library",
         "//vendor/github.com/google/gofuzz:go_default_library",

vendor/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -24,7 +24,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeserializer "k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
-	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
 )
@@ -36,14 +36,19 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			c.FuzzNoCustom(obj)
 			obj.KubernetesVersion = "v10"
 			obj.API.BindPort = 20
-			obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour}
 			obj.API.AdvertiseAddress = "foo"
 			obj.Networking.ServiceSubnet = "foo"
 			obj.Networking.DNSDomain = "foo"
 			obj.AuthorizationModes = []string{"foo"}
 			obj.CertificatesDir = "foo"
 			obj.APIServerCertSANs = []string{"foo"}
+			obj.Etcd.ServerCertSANs = []string{"foo"}
+			obj.Etcd.PeerCertSANs = []string{"foo"}
 			obj.Token = "foo"
+			obj.CRISocket = "foo"
+			obj.TokenTTL = &metav1.Duration{Duration: 1 * time.Hour}
+			obj.TokenUsages = []string{"foo"}
+			obj.TokenGroups = []string{"foo"}
 			obj.Etcd.Image = "foo"
 			obj.Etcd.DataDir = "foo"
 			obj.ImageRepository = "foo"
@@ -64,22 +69,20 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 				OperatorVersion:    "v0.1.0",
 			}
 			obj.KubeletConfiguration = kubeadm.KubeletConfiguration{
-				BaseConfig: &kubeletconfigv1alpha1.KubeletConfiguration{
-					PodManifestPath: "foo",
-					AllowPrivileged: utilpointer.BoolPtr(true),
-					ClusterDNS:      []string{"foo"},
-					ClusterDomain:   "foo",
-					Authorization:   kubeletconfigv1alpha1.KubeletAuthorization{Mode: "foo"},
-					Authentication: kubeletconfigv1alpha1.KubeletAuthentication{
-						X509: kubeletconfigv1alpha1.KubeletX509Authentication{ClientCAFile: "foo"},
+				BaseConfig: &kubeletconfigv1beta1.KubeletConfiguration{
+					StaticPodPath: "foo",
+					ClusterDNS:    []string{"foo"},
+					ClusterDomain: "foo",
+					Authorization: kubeletconfigv1beta1.KubeletAuthorization{Mode: "foo"},
+					Authentication: kubeletconfigv1beta1.KubeletAuthentication{
+						X509: kubeletconfigv1beta1.KubeletX509Authentication{ClientCAFile: "foo"},
 					},
-					CAdvisorPort: utilpointer.Int32Ptr(0),
 				},
 			}
-			kubeletconfigv1alpha1.SetDefaults_KubeletConfiguration(obj.KubeletConfiguration.BaseConfig)
+			kubeletconfigv1beta1.SetDefaults_KubeletConfiguration(obj.KubeletConfiguration.BaseConfig)
 			obj.KubeProxy = kubeadm.KubeProxy{
 				Config: &kubeproxyconfigv1alpha1.KubeProxyConfiguration{
-					FeatureGates:       "foo",
+					FeatureGates:       map[string]bool{"foo": true},
 					BindAddress:        "foo",
 					HealthzBindAddress: "foo:10256",
 					MetricsBindAddress: "foo:",
@@ -112,16 +115,21 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 					ConfigSyncPeriod: metav1.Duration{Duration: 1},
 				},
 			}
+			obj.AuditPolicyConfiguration = kubeadm.AuditPolicyConfiguration{
+				Path:      "foo",
+				LogDir:    "/foo",
+				LogMaxAge: utilpointer.Int32Ptr(0),
+			}
 		},
 		func(obj *kubeadm.NodeConfiguration, c fuzz.Continue) {
 			c.FuzzNoCustom(obj)
 			obj.CACertPath = "foo"
-			obj.CACertPath = "foo"
 			obj.DiscoveryFile = "foo"
 			obj.DiscoveryToken = "foo"
 			obj.DiscoveryTokenAPIServers = []string{"foo"}
 			obj.TLSBootstrapToken = "foo"
 			obj.Token = "foo"
+			obj.CRISocket = "foo"
 		},
 	}
 }

vendor/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/install/BUILD

@@ -39,8 +39,7 @@ filegroup(
 go_test(
     name = "go_default_test",
     srcs = ["install_test.go"],
-    importpath = "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/install",
-    library = ":go_default_library",
+    embed = [":go_default_library"],
     deps = [
         "//cmd/kubeadm/app/apis/kubeadm/fuzzer:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/testing/roundtrip:go_default_library",

vendor/k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/types.go

@@ -17,8 +17,9 @@ limitations under the License.
 package kubeadm
 
 import (
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	kubeletconfigv1alpha1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1alpha1"
+	kubeletconfigv1beta1 "k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig/v1beta1"
 	kubeproxyconfigv1alpha1 "k8s.io/kubernetes/pkg/proxy/apis/kubeproxyconfig/v1alpha1"
 )
 
@@ -29,87 +30,172 @@ import (
 type MasterConfiguration struct {
 	metav1.TypeMeta
 
-	API                  API
-	KubeProxy            KubeProxy
-	Etcd                 Etcd
+	// API holds configuration for the k8s apiserver.
+	API API
+	// KubeProxy holds configuration for the k8s service proxy.
+	KubeProxy KubeProxy
+	// Etcd holds configuration for etcd.
+	Etcd Etcd
+	// KubeletConfiguration holds configuration for the kubelet.
 	KubeletConfiguration KubeletConfiguration
-	Networking           Networking
-	KubernetesVersion    string
-	CloudProvider        string
-	NodeName             string
-	AuthorizationModes   []string
+	// Networking holds configuration for the networking topology of the cluster.
+	Networking Networking
+	// KubernetesVersion is the target version of the control plane.
+	KubernetesVersion string
+	// CloudProvider is the name of the cloud provider.
+	CloudProvider string
+	// NodeName is the name of the node that will host the k8s control plane.
+	// Defaults to the hostname if not provided.
+	NodeName string
+	// AuthorizationModes is a set of authorization modes used inside the cluster.
+	// If not specified, defaults to Node and RBAC, meaning both the node
+	// authorizer and RBAC are enabled.
+	AuthorizationModes []string
+	// NoTaintMaster will, if set, suppress the tainting of the
+	// master node allowing workloads to be run on it (e.g. in
+	// single node configurations).
+	NoTaintMaster bool
 
-	Token    string
+	// Mark the controller and api server pods as privileged as some cloud
+	// controllers like openstack need escalated privileges under some conditions
+	// example - loading a config drive to fetch node information.
+	PrivilegedPods bool
+
+	// Token is used for establishing bidirectional trust between nodes and masters.
+	// Used for joining nodes in the cluster.
+	Token string
+	// TokenTTL defines the ttl for Token. Defaults to 24h.
 	TokenTTL *metav1.Duration
+	// TokenUsages describes the ways in which this token can be used.
+	TokenUsages []string
+	// Extra groups that this token will authenticate as when used for authentication
+	TokenGroups []string
 
-	APIServerExtraArgs         map[string]string
+	// CRISocket is used to retrieve container runtime info.
+	CRISocket string
+
+	// APIServerExtraArgs is a set of extra flags to pass to the API Server or override
+	// default ones in form of <flagname>=<value>.
+	// TODO: This is temporary and ideally we would like to switch all components to
+	// use ComponentConfig + ConfigMaps.
+	APIServerExtraArgs map[string]string
+	// ControllerManagerExtraArgs is a set of extra flags to pass to the Controller Manager
+	// or override default ones in form of <flagname>=<value>
+	// TODO: This is temporary and ideally we would like to switch all components to
+	// use ComponentConfig + ConfigMaps.
 	ControllerManagerExtraArgs map[string]string
-	SchedulerExtraArgs         map[string]string
+	// SchedulerExtraArgs is a set of extra flags to pass to the Scheduler or override
+	// default ones in form of <flagname>=<value>
+	// TODO: This is temporary and ideally we would like to switch all components to
+	// use ComponentConfig + ConfigMaps.
+	SchedulerExtraArgs map[string]string
 
-	APIServerExtraVolumes         []HostPathMount
+	// APIServerExtraVolumes is an extra set of host volumes mounted to the API server.
+	APIServerExtraVolumes []HostPathMount
+	// ControllerManagerExtraVolumes is an extra set of host volumes mounted to the
+	// Controller Manager.
 	ControllerManagerExtraVolumes []HostPathMount
-	SchedulerExtraVolumes         []HostPathMount
+	// SchedulerExtraVolumes is an extra set of host volumes mounted to the scheduler.
+	SchedulerExtraVolumes []HostPathMount
 
-	// APIServerCertSANs sets extra Subject Alternative Names for the API Server signing cert
+	// APIServerCertSANs sets extra Subject Alternative Names for the API Server
+	// signing cert.
 	APIServerCertSANs []string
-	// CertificatesDir specifies where to store or look for all required certificates
+	// CertificatesDir specifies where to store or look for all required certificates.
 	CertificatesDir string
 
-	// ImageRepository what container registry to pull control plane images from
+	// ImagePullPolicy for control plane images. Can be Always, IfNotPresent or Never.
+	ImagePullPolicy v1.PullPolicy
+
+	// ImageRepository is the container registry to pull control plane images from.
 	ImageRepository string
 
-	// Container registry for core images generated by CI
+	// CIImageRepository is the container registry for core images generated by CI.
+	// Useful for running kubeadm with images from CI builds.
 	// +k8s:conversion-gen=false
 	CIImageRepository string
 
-	// UnifiedControlPlaneImage specifies if a specific container image should be used for all control plane components
+	// UnifiedControlPlaneImage specifies if a specific container image should be
+	// used for all control plane components.
 	UnifiedControlPlaneImage string
 
-	// FeatureGates enabled by the user
+	// AuditPolicyConfiguration defines the options for the api server audit system.
+	AuditPolicyConfiguration AuditPolicyConfiguration
+
+	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool
 }
 
 // API struct contains elements of API server address.
 type API struct {
-	// AdvertiseAddress sets the address for the API server to advertise.
+	// AdvertiseAddress sets the IP address for the API server to advertise.
 	AdvertiseAddress string
-	// BindPort sets the secure port for the API Server to bind to
+	// ControlPlaneEndpoint sets the DNS address for the API server
+	ControlPlaneEndpoint string
+	// BindPort sets the secure port for the API Server to bind to.
+	// Defaults to 6443.
 	BindPort int32
 }
 
-// TokenDiscovery contains elements needed for token discovery
+// TokenDiscovery contains elements needed for token discovery.
 type TokenDiscovery struct {
-	ID        string
-	Secret    string
-	Addresses []string
+	// ID is the first part of a bootstrap token. Considered public information.
+	// It is used when referring to a token without leaking the secret part.
+	ID string
+	// Secret is the second part of a bootstrap token. Should only be shared
+	// with trusted parties.
+	Secret string
+	// TODO: Seems unused. Remove?
+	// Addresses []string
 }
 
-// Networking contains elements describing cluster's networking configuration
+// Networking contains elements describing cluster's networking configuration.
 type Networking struct {
+	// ServiceSubnet is the subnet used by k8s services. Defaults to "10.96.0.0/12".
 	ServiceSubnet string
-	PodSubnet     string
-	DNSDomain     string
+	// PodSubnet is the subnet used by pods.
+	PodSubnet string
+	// DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local".
+	DNSDomain string
 }
 
-// Etcd contains elements describing Etcd configuration
+// Etcd contains elements describing Etcd configuration.
 type Etcd struct {
+	// Endpoints of etcd members. Useful for using external etcd.
+	// If not provided, kubeadm will run etcd in a static pod.
 	Endpoints []string
-	CAFile    string
-	CertFile  string
-	KeyFile   string
-	DataDir   string
+	// CAFile is an SSL Certificate Authority file used to secure etcd communication.
+	CAFile string
+	// CertFile is an SSL certification file used to secure etcd communication.
+	CertFile string
+	// KeyFile is an SSL key file used to secure etcd communication.
+	KeyFile string
+	// DataDir is the directory etcd will place its data.
+	// Defaults to "/var/lib/etcd".
+	DataDir string
+	// ExtraArgs are extra arguments provided to the etcd binary
+	// when run inside a static pod.
 	ExtraArgs map[string]string
-	// Image specifies which container image to use for running etcd. If empty, automatically populated by kubeadm using the image repository and default etcd version
-	Image      string
+	// Image specifies which container image to use for running etcd.
+	// If empty, automatically populated by kubeadm using the image
+	// repository and default etcd version.
+	Image string
+	// SelfHosted holds configuration for self-hosting etcd.
 	SelfHosted *SelfHostedEtcd
+	// ServerCertSANs sets extra Subject Alternative Names for the etcd server
+	// signing cert. This is currently used for the etcd static-pod.
+	ServerCertSANs []string
+	// PeerCertSANs sets extra Subject Alternative Names for the etcd peer
+	// signing cert. This is currently used for the etcd static-pod.
+	PeerCertSANs []string
 }
 
-// SelfHostedEtcd describes options required to configure self-hosted etcd
+// SelfHostedEtcd describes options required to configure self-hosted etcd.
 type SelfHostedEtcd struct {
-	// CertificatesDir represents the directory where all etcd TLS assets are stored. By default this is
-	// a dir names "etcd" in the main CertificatesDir value.
+	// CertificatesDir represents the directory where all etcd TLS assets are stored.
+	// Defaults to "/etc/kubernetes/pki/etcd".
 	CertificatesDir string
-	// ClusterServiceName is the name of the service that load balances the etcd cluster
+	// ClusterServiceName is the name of the service that load balances the etcd cluster.
 	ClusterServiceName string
 	// EtcdVersion is the version of etcd running in the cluster.
 	EtcdVersion string
@@ -119,18 +205,35 @@ type SelfHostedEtcd struct {
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
-// NodeConfiguration contains elements describing a particular node
+// NodeConfiguration contains elements describing a particular node.
+// TODO: This struct should be replaced by dynamic kubelet configuration.
 type NodeConfiguration struct {
 	metav1.TypeMeta
 
-	CACertPath     string
-	DiscoveryFile  string
+	// CACertPath is the path to the SSL certificate authority used to
+	// secure comunications between node and master.
+	// Defaults to "/etc/kubernetes/pki/ca.crt".
+	CACertPath string
+	// DiscoveryFile is a file or url to a kubeconfig file from which to
+	// load cluster information.
+	DiscoveryFile string
+	// DiscoveryToken is a token used to validate cluster information
+	// fetched from the master.
 	DiscoveryToken string
-	// Currently we only pay attention to one api server but hope to support >1 in the future
+	// DiscoveryTokenAPIServers is a set of IPs to API servers from which info
+	// will be fetched. Currently we only pay attention to one API server but
+	// hope to support >1 in the future.
 	DiscoveryTokenAPIServers []string
-	NodeName                 string
-	TLSBootstrapToken        string
-	Token                    string
+	// NodeName is the name of the node to join the cluster. Defaults
+	// to the name of the host.
+	NodeName string
+	// TLSBootstrapToken is a token used for TLS bootstrapping.
+	// Defaults to Token.
+	TLSBootstrapToken string
+	// Token is used for both discovery and TLS bootstrapping.
+	Token string
+	// CRISocket is used to retrieve container runtime info.
+	CRISocket string
 
 	// DiscoveryTokenCACertHashes specifies a set of public key pins to verify
 	// when token-based discovery is used. The root CA found during discovery
@@ -147,13 +250,13 @@ type NodeConfiguration struct {
 	// the security of kubeadm since other nodes can impersonate the master.
 	DiscoveryTokenUnsafeSkipCAVerification bool
 
-	// FeatureGates enabled by the user
+	// FeatureGates enabled by the user.
 	FeatureGates map[string]bool
 }
 
-// KubeletConfiguration contains elements describing initial remote configuration of kubelet
+// KubeletConfiguration contains elements describing initial remote configuration of kubelet.
 type KubeletConfiguration struct {
-	BaseConfig *kubeletconfigv1alpha1.KubeletConfiguration
+	BaseConfig *kubeletconfigv1beta1.KubeletConfiguration
 }
 
 // GetControlPlaneImageRepository returns name of image repository
@@ -169,14 +272,29 @@ func (cfg *MasterConfiguration) GetControlPlaneImageRepository() string {
 }
 
 // HostPathMount contains elements describing volumes that are mounted from the
-// host
+// host.
 type HostPathMount struct {
-	Name      string
-	HostPath  string
+	// Name of the volume inside the pod template.
+	Name string
+	// HostPath is the path in the host that will be mounted inside
+	// the pod.
+	HostPath string
+	// MountPath is the path inside the pod where hostPath will be mounted.
 	MountPath string
 }
 
-// KubeProxy contains elements describing the proxy configuration
+// KubeProxy contains elements describing the proxy configuration.
 type KubeProxy struct {
 	Config *kubeproxyconfigv1alpha1.KubeProxyConfiguration
 }
+
+// AuditPolicyConfiguration holds the options for configuring the api server audit policy.
+type AuditPolicyConfiguration struct {
+	// Path is the local path to an audit policy.
+	Path string
+	// LogDir is the local path to the directory where logs should be stored.
+	LogDir string
+	// LogMaxAge is the number of days logs will be stored for. 0 indicates forever.
+	LogMaxAge *int32
+	//TODO(chuckha) add other options for audit policy.
+}