mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 10:33:35 +00:00
vendor update for CSI 0.3.0
This commit is contained in:
gman
71
vendor/k8s.io/kubernetes/cluster/gce/gci/BUILD
generated
vendored
Normal file
71
vendor/k8s.io/kubernetes/cluster/gce/gci/BUILD
generated
vendored
Normal file
@ -0,0 +1,71 @@
|
||||
load("@io_bazel_rules_go//go:def.bzl", "go_test")
|
||||
load("@io_kubernetes_build//defs:pkg.bzl", "pkg_tar")
|
||||
load("@io_kubernetes_build//defs:build.bzl", "release_filegroup")
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = [
|
||||
"apiserver_manifest_test.go",
|
||||
"configure_helper_test.go",
|
||||
],
|
||||
data = [
|
||||
":scripts-test-data",
|
||||
"//cluster/gce/manifests",
|
||||
],
|
||||
deps = [
|
||||
"//pkg/api/legacyscheme:go_default_library",
|
||||
"//vendor/k8s.io/api/core/v1:go_default_library",
|
||||
"//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
|
||||
],
|
||||
)
|
||||
|
||||
# Having the COS code from the GCE cluster deploy hosted with the release is
|
||||
# useful for GKE. This list should match the list in
|
||||
# kubernetes/release/lib/releaselib.sh.
|
||||
release_filegroup(
|
||||
name = "gcs-release-artifacts",
|
||||
srcs = [
|
||||
"configure.sh",
|
||||
"master.yaml",
|
||||
"node.yaml",
|
||||
"shutdown.sh",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
pkg_tar(
|
||||
name = "gci-trusty-manifests",
|
||||
srcs = glob(["gke-internal-configure-helper.sh"]),
|
||||
files = {
|
||||
"//cluster/gce/gci/mounter": "gci-mounter",
|
||||
"configure-helper.sh": "gci-configure-helper.sh",
|
||||
"health-monitor.sh": "health-monitor.sh",
|
||||
},
|
||||
mode = "0755",
|
||||
strip_prefix = ".",
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "package-srcs",
|
||||
srcs = glob(["**"]),
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:private"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "all-srcs",
|
||||
srcs = [
|
||||
":package-srcs",
|
||||
"//cluster/gce/gci/mounter:all-srcs",
|
||||
],
|
||||
tags = ["automanaged"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "scripts-test-data",
|
||||
srcs = [
|
||||
"configure-helper.sh",
|
||||
],
|
||||
)
|
||||
212
vendor/k8s.io/kubernetes/cluster/gce/gci/apiserver_manifest_test.go
generated
vendored
Normal file
212
vendor/k8s.io/kubernetes/cluster/gce/gci/apiserver_manifest_test.go
generated
vendored
Normal file
@ -0,0 +1,212 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
/*
|
||||
Template for defining the environment state of configure-helper.sh
|
||||
The environment of configure-helper.sh is initially configured via kube-env file. However, as deploy-helper
|
||||
executes new variables are created. ManifestTestCase does not care where a variable came from. However, future
|
||||
test scenarios, may require such a distinction.
|
||||
The list of variables is, by no means, complete - this is what is required to run currently defined tests.
|
||||
*/
|
||||
deployHelperEnv = `
|
||||
readonly KUBE_HOME={{.KubeHome}}
|
||||
readonly KUBE_API_SERVER_LOG_PATH=${KUBE_HOME}/kube-apiserver.log
|
||||
readonly KUBE_API_SERVER_AUDIT_LOG_PATH=${KUBE_HOME}/kube-apiserver-audit.log
|
||||
readonly CLOUD_CONFIG_OPT=--cloud-config=/etc/gce.conf
|
||||
readonly CA_CERT_BUNDLE_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_CERT_PATH=/foo/bar
|
||||
readonly APISERVER_SERVER_KEY_PATH=/foo/bar
|
||||
readonly APISERVER_CLIENT_CERT_PATH=/foo/bar
|
||||
readonly CLOUD_CONFIG_MOUNT="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true},"
|
||||
readonly CLOUD_CONFIG_VOLUME="{\"name\": \"cloudconfigmount\",\"hostPath\": {\"path\": \"/etc/gce.conf\", \"type\": \"FileOrCreate\"}},"
|
||||
readonly DOCKER_REGISTRY="k8s.gcr.io"
|
||||
readonly ENABLE_LEGACY_ABAC=false
|
||||
readonly ETC_MANIFESTS=${KUBE_HOME}/etc/kubernetes/manifests
|
||||
readonly KUBE_API_SERVER_DOCKER_TAG=v1.11.0-alpha.0.1808_3c7452dc11645d-dirty
|
||||
readonly LOG_OWNER_USER=$(whoami)
|
||||
readonly LOG_OWNER_GROUP=$(id -gn)
|
||||
ENCRYPTION_PROVIDER_CONFIG={{.EncryptionProviderConfig}}
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH={{.EncryptionProviderConfigPath}}
|
||||
readonly ETCD_KMS_KEY_ID={{.ETCDKMSKeyID}}
|
||||
`
|
||||
kubeAPIServerManifestFileName = "kube-apiserver.manifest"
|
||||
kmsPluginManifestFileName = "kms-plugin-container.manifest"
|
||||
kubeAPIServerStartFuncName = "start-kube-apiserver"
|
||||
|
||||
// Position of containers within a pod manifest
|
||||
kmsPluginContainerIndex = 0
|
||||
apiServerContainerIndexNoKMS = 0
|
||||
apiServerContainerIndexWithKMS = 1
|
||||
|
||||
// command": [
|
||||
// "/bin/sh", - Index 0
|
||||
// "-c", - Index 1
|
||||
// "exec /usr/local/bin/kube-apiserver " - Index 2
|
||||
execArgsIndex = 2
|
||||
|
||||
socketVolumeMountIndexKMSPlugin = 1
|
||||
socketVolumeMountIndexAPIServer = 0
|
||||
)
|
||||
|
||||
type kubeAPIServerEnv struct {
|
||||
KubeHome string
|
||||
EncryptionProviderConfig string
|
||||
EncryptionProviderConfigPath string
|
||||
ETCDKMSKeyID string
|
||||
}
|
||||
|
||||
type kubeAPIServerManifestTestCase struct {
|
||||
*ManifestTestCase
|
||||
apiServerContainer v1.Container
|
||||
kmsPluginContainer v1.Container
|
||||
}
|
||||
|
||||
func newKubeAPIServerManifestTestCase(t *testing.T) *kubeAPIServerManifestTestCase {
|
||||
return &kubeAPIServerManifestTestCase{
|
||||
ManifestTestCase: newManifestTestCase(t, kubeAPIServerManifestFileName, kubeAPIServerStartFuncName, []string{kmsPluginManifestFileName}),
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) mustLoadContainers() {
|
||||
c.mustLoadPodFromManifest()
|
||||
|
||||
switch len(c.pod.Spec.Containers) {
|
||||
case 1:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexNoKMS]
|
||||
case 2:
|
||||
c.apiServerContainer = c.pod.Spec.Containers[apiServerContainerIndexWithKMS]
|
||||
c.kmsPluginContainer = c.pod.Spec.Containers[kmsPluginContainerIndex]
|
||||
default:
|
||||
c.t.Fatalf("got %d containers in apiserver pod, want 1 or 2", len(c.pod.Spec.Containers))
|
||||
}
|
||||
}
|
||||
|
||||
func (c *kubeAPIServerManifestTestCase) invokeTest(e kubeAPIServerEnv) {
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
c.mustLoadContainers()
|
||||
}
|
||||
|
||||
func getEncryptionProviderConfigFlag(path string) string {
|
||||
return fmt.Sprintf("--experimental-encryption-provider-config=%s", path)
|
||||
}
|
||||
|
||||
func TestEncryptionProviderFlag(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfig: base64.StdEncoding.EncodeToString([]byte("FOO")),
|
||||
EncryptionProviderConfigPath: p,
|
||||
}
|
||||
|
||||
c.mustInvokeFunc(deployHelperEnv, e)
|
||||
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestKMSEncryptionProviderConfig asserts that if ETCD_KMS_KEY_ID is set then start-kube-apiserver will produce
|
||||
// EncryptionProviderConfig file of type KMS and inject experimental-encryption-provider-config startup flag.
|
||||
func TestKMSEncryptionProviderConfig(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
e := kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
expectedFlag := getEncryptionProviderConfigFlag(e.EncryptionProviderConfigPath)
|
||||
execArgs := c.apiServerContainer.Command[execArgsIndex]
|
||||
if !strings.Contains(execArgs, expectedFlag) {
|
||||
c.t.Fatalf("Got %q, wanted the flag to contain %q", execArgs, expectedFlag)
|
||||
}
|
||||
|
||||
p := filepath.Join(c.kubeHome, "encryption-provider-config.yaml")
|
||||
if _, err := os.Stat(p); err != nil {
|
||||
c.t.Fatalf("Expected encryption provider config to be written to %s, but stat failed with error: %v", p, err)
|
||||
}
|
||||
|
||||
d, err := ioutil.ReadFile(p)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read encryption provider config %s", p)
|
||||
}
|
||||
|
||||
if !strings.Contains(string(d), "name: grpc-kms-provider") {
|
||||
c.t.Fatalf("Got %s\n, wanted encryption provider config to be of type grpc-kms", string(d))
|
||||
}
|
||||
}
|
||||
|
||||
func TestKMSPluginAndAPIServerSharedVolume(t *testing.T) {
|
||||
c := newKubeAPIServerManifestTestCase(t)
|
||||
defer c.tearDown()
|
||||
|
||||
var e = kubeAPIServerEnv{
|
||||
KubeHome: c.kubeHome,
|
||||
EncryptionProviderConfigPath: filepath.Join(c.kubeHome, "encryption-provider-config.yaml"),
|
||||
ETCDKMSKeyID: "FOO",
|
||||
}
|
||||
|
||||
c.invokeTest(e)
|
||||
|
||||
k := c.kmsPluginContainer.VolumeMounts[socketVolumeMountIndexKMSPlugin].MountPath
|
||||
a := c.apiServerContainer.VolumeMounts[socketVolumeMountIndexAPIServer].MountPath
|
||||
|
||||
if k != a {
|
||||
t.Fatalf("Got %s!=%s, wanted KMSPlugin VolumeMount #1:%s to be equal to kube-apiserver VolumeMount #0:%s",
|
||||
k, a, k, a)
|
||||
}
|
||||
}
|
||||
601
vendor/k8s.io/kubernetes/cluster/gce/gci/configure-helper.sh
generated
vendored
601
vendor/k8s.io/kubernetes/cluster/gce/gci/configure-helper.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -27,6 +27,8 @@ set -o pipefail
|
||||
|
||||
readonly UUID_MNT_PREFIX="/mnt/disks/by-uuid/google-local-ssds"
|
||||
readonly UUID_BLOCK_PREFIX="/dev/disk/by-uuid/google-local-ssds"
|
||||
readonly COREDNS_AUTOSCALER="Deployment/coredns"
|
||||
readonly KUBEDNS_AUTOSCALER="Deployment/kube-dns"
|
||||
|
||||
# Use --retry-connrefused opt only if it's supported by curl.
|
||||
CURL_RETRY_CONNREFUSED=""
|
||||
@ -183,6 +185,7 @@ function safe-format-and-mount() {
|
||||
mkdir -p "${mountpoint}"
|
||||
echo "Mounting '${device}' at '${mountpoint}'"
|
||||
mount -o discard,defaults "${device}" "${mountpoint}"
|
||||
chmod a+w "${mountpoint}"
|
||||
}
|
||||
|
||||
# Gets a devices UUID and bind mounts the device to mount location in
|
||||
@ -542,6 +545,9 @@ function create-master-auth {
|
||||
if [[ -n "${KUBE_SCHEDULER_TOKEN:-}" ]]; then
|
||||
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_SCHEDULER_TOKEN}," "system:kube-scheduler,uid:system:kube-scheduler"
|
||||
fi
|
||||
if [[ -n "${KUBE_CLUSTER_AUTOSCALER_TOKEN:-}" ]]; then
|
||||
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_CLUSTER_AUTOSCALER_TOKEN}," "cluster-autoscaler,uid:cluster-autoscaler"
|
||||
fi
|
||||
if [[ -n "${KUBE_PROXY_TOKEN:-}" ]]; then
|
||||
append_or_replace_prefixed_line "${known_tokens_csv}" "${KUBE_PROXY_TOKEN}," "system:kube-proxy,uid:kube_proxy"
|
||||
fi
|
||||
@ -889,8 +895,9 @@ function create-kubelet-kubeconfig() {
|
||||
echo "Must provide API server address to create Kubelet kubeconfig file!"
|
||||
exit 1
|
||||
fi
|
||||
echo "Creating kubelet kubeconfig file"
|
||||
cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
|
||||
if [[ "${CREATE_BOOTSTRAP_KUBECONFIG:-true}" == "true" ]]; then
|
||||
echo "Creating kubelet bootstrap-kubeconfig file"
|
||||
cat <<EOF >/var/lib/kubelet/bootstrap-kubeconfig
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
users:
|
||||
@ -910,6 +917,13 @@ contexts:
|
||||
name: service-account-context
|
||||
current-context: service-account-context
|
||||
EOF
|
||||
elif [[ "${FETCH_BOOTSTRAP_KUBECONFIG:-false}" == "true" ]]; then
|
||||
echo "Fetching kubelet bootstrap-kubeconfig file from metadata"
|
||||
get-metadata-value "instance/attributes/bootstrap-kubeconfig" >/var/lib/kubelet/bootstrap-kubeconfig
|
||||
else
|
||||
echo "Fetching kubelet kubeconfig file from metadata"
|
||||
get-metadata-value "instance/attributes/kubeconfig" >/var/lib/kubelet/kubeconfig
|
||||
fi
|
||||
}
|
||||
|
||||
# Uses KUBELET_CA_CERT (falling back to CA_CERT), KUBELET_CERT, and KUBELET_KEY
|
||||
@ -995,6 +1009,30 @@ current-context: kube-scheduler
|
||||
EOF
|
||||
}
|
||||
|
||||
function create-clusterautoscaler-kubeconfig {
|
||||
echo "Creating cluster-autoscaler kubeconfig file"
|
||||
mkdir -p /etc/srv/kubernetes/cluster-autoscaler
|
||||
cat <<EOF >/etc/srv/kubernetes/cluster-autoscaler/kubeconfig
|
||||
apiVersion: v1
|
||||
kind: Config
|
||||
users:
|
||||
- name: cluster-autoscaler
|
||||
user:
|
||||
token: ${KUBE_CLUSTER_AUTOSCALER_TOKEN}
|
||||
clusters:
|
||||
- name: local
|
||||
cluster:
|
||||
insecure-skip-tls-verify: true
|
||||
server: https://localhost:443
|
||||
contexts:
|
||||
- context:
|
||||
cluster: local
|
||||
user: cluster-autoscaler
|
||||
name: cluster-autoscaler
|
||||
current-context: cluster-autoscaler
|
||||
EOF
|
||||
}
|
||||
|
||||
function create-kubescheduler-policy-config {
|
||||
echo "Creating kube-scheduler policy config file"
|
||||
mkdir -p /etc/srv/kubernetes/kube-scheduler
|
||||
@ -1112,7 +1150,8 @@ function start-kubelet {
|
||||
echo "Using kubelet binary at ${kubelet_bin}"
|
||||
|
||||
local -r kubelet_env_file="/etc/default/kubelet"
|
||||
echo "KUBELET_OPTS=\"${KUBELET_ARGS}\"" > "${kubelet_env_file}"
|
||||
local kubelet_opts="${KUBELET_ARGS} ${KUBELET_CONFIG_FILE_ARG:-}"
|
||||
echo "KUBELET_OPTS=\"${kubelet_opts}\"" > "${kubelet_env_file}"
|
||||
|
||||
# Write the systemd service file for kubelet.
|
||||
cat <<EOF >/etc/systemd/system/kubelet.service
|
||||
@ -1131,6 +1170,7 @@ ExecStart=${kubelet_bin} \$KUBELET_OPTS
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl start kubelet.service
|
||||
}
|
||||
|
||||
@ -1142,13 +1182,18 @@ function start-node-problem-detector {
|
||||
local -r km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor.json"
|
||||
# TODO(random-liu): Handle this for alternative container runtime.
|
||||
local -r dm_config="${KUBE_HOME}/node-problem-detector/config/docker-monitor.json"
|
||||
local -r custom_km_config="${KUBE_HOME}/node-problem-detector/config/kernel-monitor-counter.json"
|
||||
echo "Using node problem detector binary at ${npd_bin}"
|
||||
local flags="${NPD_TEST_LOG_LEVEL:-"--v=2"} ${NPD_TEST_ARGS:-}"
|
||||
flags+=" --logtostderr"
|
||||
flags+=" --system-log-monitors=${km_config},${dm_config}"
|
||||
flags+=" --custom-plugin-monitors=${custom_km_config}"
|
||||
flags+=" --apiserver-override=https://${KUBERNETES_MASTER_NAME}?inClusterConfig=false&auth=/var/lib/node-problem-detector/kubeconfig"
|
||||
local -r npd_port=${NODE_PROBLEM_DETECTOR_PORT:-20256}
|
||||
flags+=" --port=${npd_port}"
|
||||
if [[ -n "${EXTRA_NPD_ARGS:-}" ]]; then
|
||||
flags+=" ${EXTRA_NPD_ARGS}"
|
||||
fi
|
||||
|
||||
# Write the systemd service file for node problem detector.
|
||||
cat <<EOF >/etc/systemd/system/node-problem-detector.service
|
||||
@ -1175,7 +1220,7 @@ EOF
|
||||
function prepare-log-file {
|
||||
touch $1
|
||||
chmod 644 $1
|
||||
chown root:root $1
|
||||
chown "${LOG_OWNER_USER:-root}":"${LOG_OWNER_GROUP:-root}" $1
|
||||
}
|
||||
|
||||
# Prepares parameters for kube-proxy manifest.
|
||||
@ -1195,7 +1240,15 @@ function prepare-kube-proxy-manifest-variables {
|
||||
params+=" --feature-gates=${FEATURE_GATES}"
|
||||
fi
|
||||
if [[ "${KUBE_PROXY_MODE:-}" == "ipvs" ]];then
|
||||
params+=" --proxy-mode=ipvs --feature-gates=SupportIPVSProxyMode=true"
|
||||
sudo modprobe -a ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack_ipv4
|
||||
if [[ $? -eq 0 ]];
|
||||
then
|
||||
params+=" --proxy-mode=ipvs"
|
||||
else
|
||||
# If IPVS modules are not present, make sure the node does not come up as
|
||||
# healthy.
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
params+=" --iptables-sync-period=1m --iptables-min-sync-period=10s --ipvs-sync-period=1m --ipvs-min-sync-period=10s"
|
||||
if [[ -n "${KUBEPROXY_TEST_ARGS:-}" ]]; then
|
||||
@ -1209,10 +1262,6 @@ function prepare-kube-proxy-manifest-variables {
|
||||
kube_cache_mutation_detector_env_name="- name: KUBE_CACHE_MUTATION_DETECTOR"
|
||||
kube_cache_mutation_detector_env_value="value: \"${ENABLE_CACHE_MUTATION_DETECTOR}\""
|
||||
fi
|
||||
local pod_priority=""
|
||||
if [[ "${ENABLE_POD_PRIORITY:-}" == "true" ]]; then
|
||||
pod_priority="priorityClassName: system-node-critical"
|
||||
fi
|
||||
sed -i -e "s@{{kubeconfig}}@${kubeconfig}@g" ${src_file}
|
||||
sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${kube_docker_registry}@g" ${src_file}
|
||||
sed -i -e "s@{{pillar\['kube-proxy_docker_tag'\]}}@${kube_proxy_docker_tag}@g" ${src_file}
|
||||
@ -1220,7 +1269,6 @@ function prepare-kube-proxy-manifest-variables {
|
||||
sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
||||
sed -i -e "s@{{kube_cache_mutation_detector_env_name}}@${kube_cache_mutation_detector_env_name}@g" ${src_file}
|
||||
sed -i -e "s@{{kube_cache_mutation_detector_env_value}}@${kube_cache_mutation_detector_env_value}@g" ${src_file}
|
||||
sed -i -e "s@{{pod_priority}}@${pod_priority}@g" ${src_file}
|
||||
sed -i -e "s@{{ cpurequest }}@100m@g" ${src_file}
|
||||
sed -i -e "s@{{api_servers_with_port}}@${api_servers}@g" ${src_file}
|
||||
sed -i -e "s@{{kubernetes_service_host_env_value}}@${KUBERNETES_MASTER_NAME}@g" ${src_file}
|
||||
@ -1253,6 +1301,7 @@ function prepare-etcd-manifest {
|
||||
local cluster_state="new"
|
||||
local etcd_protocol="http"
|
||||
local etcd_creds=""
|
||||
local etcd_extra_args="${ETCD_EXTRA_ARGS:-}"
|
||||
|
||||
if [[ -n "${INITIAL_ETCD_CLUSTER_STATE:-}" ]]; then
|
||||
cluster_state="${INITIAL_ETCD_CLUSTER_STATE}"
|
||||
@ -1308,6 +1357,7 @@ function prepare-etcd-manifest {
|
||||
fi
|
||||
sed -i -e "s@{{ *etcd_protocol *}}@$etcd_protocol@g" "${temp_file}"
|
||||
sed -i -e "s@{{ *etcd_creds *}}@$etcd_creds@g" "${temp_file}"
|
||||
sed -i -e "s@{{ *etcd_extra_args *}}@$etcd_extra_args@g" "${temp_file}"
|
||||
if [[ -n "${ETCD_VERSION:-}" ]]; then
|
||||
sed -i -e "s@{{ *pillar\.get('etcd_version', '\(.*\)') *}}@${ETCD_VERSION}@g" "${temp_file}"
|
||||
else
|
||||
@ -1319,7 +1369,8 @@ function prepare-etcd-manifest {
|
||||
}
|
||||
|
||||
function start-etcd-empty-dir-cleanup-pod {
|
||||
cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd-empty-dir-cleanup/etcd-empty-dir-cleanup.yaml" "/etc/kubernetes/manifests"
|
||||
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/etcd-empty-dir-cleanup.yaml"
|
||||
cp "${src_file}" "/etc/kubernetes/manifests"
|
||||
}
|
||||
|
||||
# Starts etcd server pod (and etcd-events pod if needed).
|
||||
@ -1400,8 +1451,8 @@ function prepare-mounter-rootfs {
|
||||
# DOCKER_REGISTRY
|
||||
function start-kube-apiserver {
|
||||
echo "Start kubernetes api-server"
|
||||
prepare-log-file /var/log/kube-apiserver.log
|
||||
prepare-log-file /var/log/kube-apiserver-audit.log
|
||||
prepare-log-file "${KUBE_API_SERVER_LOG_PATH:-/var/log/kube-apiserver.log}"
|
||||
prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}"
|
||||
|
||||
# Calculate variables and assemble the command line.
|
||||
local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
|
||||
@ -1453,7 +1504,9 @@ function start-kube-apiserver {
|
||||
fi
|
||||
if [[ -n "${NUM_NODES:-}" ]]; then
|
||||
# If the cluster is large, increase max-requests-inflight limit in apiserver.
|
||||
if [[ "${NUM_NODES}" -ge 1000 ]]; then
|
||||
if [[ "${NUM_NODES}" -ge 3000 ]]; then
|
||||
params+=" --max-requests-inflight=3000 --max-mutating-requests-inflight=1000"
|
||||
elif [[ "${NUM_NODES}" -ge 1000 ]]; then
|
||||
params+=" --max-requests-inflight=1500 --max-mutating-requests-inflight=500"
|
||||
fi
|
||||
# Set amount of memory available for apiserver based on number of nodes.
|
||||
@ -1515,6 +1568,33 @@ function start-kube-apiserver {
|
||||
# grows at 10MiB/s (~30K QPS), it will rotate after ~6 years if apiserver
|
||||
# never restarts. Please manually restart apiserver before this time.
|
||||
params+=" --audit-log-maxsize=2000000000"
|
||||
|
||||
# Batching parameters
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_MODE:-}" ]]; then
|
||||
params+=" --audit-log-mode=${ADVANCED_AUDIT_LOG_MODE}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_BUFFER_SIZE:-}" ]]; then
|
||||
params+=" --audit-log-batch-buffer-size=${ADVANCED_AUDIT_LOG_BUFFER_SIZE}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE:-}" ]]; then
|
||||
params+=" --audit-log-batch-max-size=${ADVANCED_AUDIT_LOG_MAX_BATCH_SIZE}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT:-}" ]]; then
|
||||
params+=" --audit-log-batch-max-wait=${ADVANCED_AUDIT_LOG_MAX_BATCH_WAIT}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_QPS:-}" ]]; then
|
||||
params+=" --audit-log-batch-throttle-qps=${ADVANCED_AUDIT_LOG_THROTTLE_QPS}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_THROTTLE_BURST:-}" ]]; then
|
||||
params+=" --audit-log-batch-throttle-burst=${ADVANCED_AUDIT_LOG_THROTTLE_BURST}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF:-}" ]]; then
|
||||
params+=" --audit-log-initial-backoff=${ADVANCED_AUDIT_LOG_INITIAL_BACKOFF}"
|
||||
fi
|
||||
# Truncating backend parameters
|
||||
if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then
|
||||
params+=" --audit-log-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}"
|
||||
fi
|
||||
fi
|
||||
if [[ "${ADVANCED_AUDIT_BACKEND:-}" == *"webhook"* ]]; then
|
||||
params+=" --audit-webhook-mode=batch"
|
||||
@ -1522,6 +1602,14 @@ function start-kube-apiserver {
|
||||
# Create the audit webhook config file, and mount it into the apiserver pod.
|
||||
local -r audit_webhook_config_file="/etc/audit_webhook.config"
|
||||
params+=" --audit-webhook-config-file=${audit_webhook_config_file}"
|
||||
create-master-audit-webhook-config "${audit_webhook_config_file}"
|
||||
audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true},"
|
||||
audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}},"
|
||||
|
||||
# Batching parameters
|
||||
if [[ -n "${ADVANCED_AUDIT_WEBHOOK_MODE:-}" ]]; then
|
||||
params+=" --audit-webhook-mode=${ADVANCED_AUDIT_WEBHOOK_MODE}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE:-}" ]]; then
|
||||
params+=" --audit-webhook-batch-buffer-size=${ADVANCED_AUDIT_WEBHOOK_BUFFER_SIZE}"
|
||||
fi
|
||||
@ -1538,17 +1626,21 @@ function start-kube-apiserver {
|
||||
params+=" --audit-webhook-batch-throttle-burst=${ADVANCED_AUDIT_WEBHOOK_THROTTLE_BURST}"
|
||||
fi
|
||||
if [[ -n "${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF:-}" ]]; then
|
||||
params+=" --audit-webhook-batch-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}"
|
||||
params+=" --audit-webhook-initial-backoff=${ADVANCED_AUDIT_WEBHOOK_INITIAL_BACKOFF}"
|
||||
fi
|
||||
# Truncating backend parameters
|
||||
if [[ -n "${ADVANCED_AUDIT_TRUNCATING_BACKEND:-}" ]]; then
|
||||
params+=" --audit-webhook-truncate-enabled=${ADVANCED_AUDIT_TRUNCATING_BACKEND}"
|
||||
fi
|
||||
create-master-audit-webhook-config "${audit_webhook_config_file}"
|
||||
audit_webhook_config_mount="{\"name\": \"auditwebhookconfigmount\",\"mountPath\": \"${audit_webhook_config_file}\", \"readOnly\": true},"
|
||||
audit_webhook_config_volume="{\"name\": \"auditwebhookconfigmount\",\"hostPath\": {\"path\": \"${audit_webhook_config_file}\", \"type\": \"FileOrCreate\"}},"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "${ENABLE_APISERVER_LOGS_HANDLER:-}" == "false" ]]; then
|
||||
params+=" --enable-logs-handler=false"
|
||||
fi
|
||||
if [[ "${APISERVER_SET_KUBELET_CA:-false}" == "true" ]]; then
|
||||
params+=" --kubelet-certificate-authority=${CA_CERT_BUNDLE_PATH}"
|
||||
fi
|
||||
|
||||
local admission_controller_config_mount=""
|
||||
local admission_controller_config_volume=""
|
||||
@ -1576,15 +1668,19 @@ function start-kube-apiserver {
|
||||
if [[ -n "${FEATURE_GATES:-}" ]]; then
|
||||
params+=" --feature-gates=${FEATURE_GATES}"
|
||||
fi
|
||||
if [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
|
||||
local -r vm_external_ip=$(curl --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --fail --silent -H 'Metadata-Flavor: Google' "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip")
|
||||
if [[ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]]; then
|
||||
params+=" --advertise-address=${MASTER_ADVERTISE_ADDRESS}"
|
||||
if [[ -n "${PROXY_SSH_USER:-}" ]]; then
|
||||
params+=" --ssh-user=${PROXY_SSH_USER}"
|
||||
params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
|
||||
fi
|
||||
elif [[ -n "${PROJECT_ID:-}" && -n "${TOKEN_URL:-}" && -n "${TOKEN_BODY:-}" && -n "${NODE_NETWORK:-}" ]]; then
|
||||
local -r vm_external_ip=$(get-metadata-value "instance/network-interfaces/0/access-configs/0/external-ip")
|
||||
if [[ -n "${PROXY_SSH_USER:-}" ]]; then
|
||||
params+=" --advertise-address=${vm_external_ip}"
|
||||
params+=" --ssh-user=${PROXY_SSH_USER}"
|
||||
params+=" --ssh-keyfile=/etc/srv/sshproxy/.sshkeyfile"
|
||||
fi
|
||||
elif [ -n "${MASTER_ADVERTISE_ADDRESS:-}" ]; then
|
||||
params="${params} --advertise-address=${MASTER_ADVERTISE_ADDRESS}"
|
||||
fi
|
||||
|
||||
local webhook_authn_config_mount=""
|
||||
@ -1623,7 +1719,7 @@ function start-kube-apiserver {
|
||||
local webhook_config_mount=""
|
||||
local webhook_config_volume=""
|
||||
if [[ -n "${GCP_AUTHZ_URL:-}" ]]; then
|
||||
authorization_mode="Webhook,${authorization_mode}"
|
||||
authorization_mode="${authorization_mode},Webhook"
|
||||
params+=" --authorization-webhook-config-file=/etc/gcp_authz.config"
|
||||
webhook_config_mount="{\"name\": \"webhookconfigmount\",\"mountPath\": \"/etc/gcp_authz.config\", \"readOnly\": false},"
|
||||
webhook_config_volume="{\"name\": \"webhookconfigmount\",\"hostPath\": {\"path\": \"/etc/gcp_authz.config\", \"type\": \"FileOrCreate\"}},"
|
||||
@ -1651,15 +1747,31 @@ function start-kube-apiserver {
|
||||
container_env="\"env\":[{${container_env}}],"
|
||||
fi
|
||||
|
||||
if [[ -n "${ETCD_KMS_KEY_ID:-}" ]]; then
|
||||
ENCRYPTION_PROVIDER_CONFIG=$(cat << EOM | base64 | tr -d '\r\n'
|
||||
kind: EncryptionConfig
|
||||
apiVersion: v1
|
||||
resources:
|
||||
- resources:
|
||||
- secrets
|
||||
providers:
|
||||
- kms:
|
||||
name: grpc-kms-provider
|
||||
cachesize: 1000
|
||||
endpoint: unix:///var/run/kmsplugin/socket.sock
|
||||
EOM
|
||||
)
|
||||
fi
|
||||
|
||||
if [[ -n "${ENCRYPTION_PROVIDER_CONFIG:-}" ]]; then
|
||||
local encryption_provider_config_path="/etc/srv/kubernetes/encryption-provider-config.yml"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${encryption_provider_config_path}"
|
||||
params+=" --experimental-encryption-provider-config=${encryption_provider_config_path}"
|
||||
ENCRYPTION_PROVIDER_CONFIG_PATH="${ENCRYPTION_PROVIDER_CONFIG_PATH:-/etc/srv/kubernetes/encryption-provider-config.yml}"
|
||||
echo "${ENCRYPTION_PROVIDER_CONFIG}" | base64 --decode > "${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
params+=" --experimental-encryption-provider-config=${ENCRYPTION_PROVIDER_CONFIG_PATH}"
|
||||
fi
|
||||
|
||||
src_file="${src_dir}/kube-apiserver.manifest"
|
||||
# Evaluate variables.
|
||||
local -r kube_apiserver_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)
|
||||
local -r kube_apiserver_docker_tag="${KUBE_API_SERVER_DOCKER_TAG:-$(cat /home/kubernetes/kube-docker-files/kube-apiserver.docker_tag)}"
|
||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||
sed -i -e "s@{{container_env}}@${container_env}@g" ${src_file}
|
||||
sed -i -e "s@{{srv_kube_path}}@/etc/srv/kubernetes@g" "${src_file}"
|
||||
@ -1686,7 +1798,68 @@ function start-kube-apiserver {
|
||||
sed -i -e "s@{{admission_controller_config_volume}}@${admission_controller_config_volume}@g" "${src_file}"
|
||||
sed -i -e "s@{{image_policy_webhook_config_mount}}@${image_policy_webhook_config_mount}@g" "${src_file}"
|
||||
sed -i -e "s@{{image_policy_webhook_config_volume}}@${image_policy_webhook_config_volume}@g" "${src_file}"
|
||||
cp "${src_file}" /etc/kubernetes/manifests
|
||||
|
||||
if [[ -z "${ETCD_KMS_KEY_ID:-}" ]]; then
|
||||
# Removing KMS related placeholders.
|
||||
sed -i -e " {
|
||||
s@{{kms_plugin_container}}@@
|
||||
|
||||
s@{{kms_socket_mount}}@@
|
||||
s@{{encryption_provider_mount}}@@
|
||||
|
||||
s@{{kms_socket_volume}}@@
|
||||
s@{{encryption_provider_volume}}@@
|
||||
} " "${src_file}"
|
||||
else
|
||||
local kms_plugin_src_file="${src_dir}/kms-plugin-container.manifest"
|
||||
|
||||
if [[ ! -f "${kms_plugin_src_file}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${kms_plugin_src_file}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ ! -f "${ENCRYPTION_PROVIDER_CONFIG_PATH}" ]]; then
|
||||
echo "Error: KMS Integration was requested, but "${ENCRYPTION_PROVIDER_CONFIG_PATH}" is missing."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# TODO: Validate that the encryption config is for KMS.
|
||||
|
||||
local kms_socket_dir="/var/run/kmsplugin"
|
||||
|
||||
# kms_socket_mnt is used by both kms_plugin and kube-apiserver - this is how these containers talk.
|
||||
local kms_socket_mnt="{ \"name\": \"kmssocket\", \"mountPath\": \"${kms_socket_dir}\", \"readOnly\": false}"
|
||||
|
||||
local kms_socket_vol="{ \"name\": \"kmssocket\", \"hostPath\": {\"path\": \"${kms_socket_dir}\", \"type\": \"DirectoryOrCreate\"}}"
|
||||
local kms_path_to_socket="${kms_socket_dir}/socket.sock"
|
||||
|
||||
local encryption_provider_mnt="{ \"name\": \"encryptionconfig\", \"mountPath\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"readOnly\": true}"
|
||||
local encryption_provider_vol="{ \"name\": \"encryptionconfig\", \"hostPath\": {\"path\": \"${ENCRYPTION_PROVIDER_CONFIG_PATH}\", \"type\": \"File\"}}"
|
||||
|
||||
# TODO these are used in other places, convert to global.
|
||||
local gce_conf_path="/etc/gce.conf"
|
||||
local cloud_config_mount="{\"name\": \"cloudconfigmount\",\"mountPath\": \"/etc/gce.conf\", \"readOnly\": true}"
|
||||
|
||||
local kms_plugin_container=$(echo $(sed " {
|
||||
s@{{kms_key_uri}}@${ETCD_KMS_KEY_ID}@
|
||||
s@{{gce_conf_path}}@${gce_conf_path}@
|
||||
s@{{kms_path_to_socket}}@${kms_path_to_socket}@
|
||||
s@{{kms_socket_mount}}@${kms_socket_mnt}@
|
||||
s@{{cloud_config_mount}}@${cloud_config_mount}@
|
||||
} " "${kms_plugin_src_file}") | tr "\n" "\\n")
|
||||
|
||||
sed -i -e " {
|
||||
s@{{kms_plugin_container}}@${kms_plugin_container},@
|
||||
|
||||
s@{{kms_socket_mount}}@${kms_socket_mnt},@
|
||||
s@{{encryption_provider_mount}}@${encryption_provider_mnt},@
|
||||
|
||||
s@{{kms_socket_volume}}@${kms_socket_vol},@
|
||||
s@{{encryption_provider_volume}}@${encryption_provider_vol},@
|
||||
} " "${src_file}"
|
||||
fi
|
||||
|
||||
cp "${src_file}" "${ETC_MANIFESTS:-/etc/kubernetes/manifests}"
|
||||
}
|
||||
|
||||
# Starts kubernetes controller manager.
|
||||
@ -1759,6 +1932,9 @@ function start-kube-controller-manager {
|
||||
params+=" --pv-recycler-pod-template-filepath-nfs=$PV_RECYCLER_OVERRIDE_TEMPLATE"
|
||||
params+=" --pv-recycler-pod-template-filepath-hostpath=$PV_RECYCLER_OVERRIDE_TEMPLATE"
|
||||
fi
|
||||
if [[ -n "${RUN_CONTROLLERS:-}" ]]; then
|
||||
params+=" --controllers=${RUN_CONTROLLERS}"
|
||||
fi
|
||||
|
||||
local -r kube_rc_docker_tag=$(cat /home/kubernetes/kube-docker-files/kube-controller-manager.docker_tag)
|
||||
local container_env=""
|
||||
@ -1830,12 +2006,15 @@ function start-kube-scheduler {
|
||||
function start-cluster-autoscaler {
|
||||
if [[ "${ENABLE_CLUSTER_AUTOSCALER:-}" == "true" ]]; then
|
||||
echo "Start kubernetes cluster autoscaler"
|
||||
setup-addon-manifests "addons" "rbac/cluster-autoscaler"
|
||||
create-clusterautoscaler-kubeconfig
|
||||
prepare-log-file /var/log/cluster-autoscaler.log
|
||||
|
||||
# Remove salt comments and replace variables with values
|
||||
local -r src_file="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/cluster-autoscaler.manifest"
|
||||
|
||||
local params="${AUTOSCALER_MIG_CONFIG} ${CLOUD_CONFIG_OPT} ${AUTOSCALER_EXPANDER_CONFIG:---expander=price}"
|
||||
params+=" --kubeconfig=/etc/srv/kubernetes/cluster-autoscaler/kubeconfig"
|
||||
sed -i -e "s@{{params}}@${params}@g" "${src_file}"
|
||||
sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
|
||||
sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
|
||||
@ -1893,6 +2072,20 @@ function download-extra-addons {
|
||||
"${curl_cmd[@]}"
|
||||
}
|
||||
|
||||
# A function that fetches a GCE metadata value and echoes it out.
|
||||
#
|
||||
# $1: URL path after /computeMetadata/v1/ (without heading slash).
|
||||
function get-metadata-value {
|
||||
curl \
|
||||
--retry 5 \
|
||||
--retry-delay 3 \
|
||||
${CURL_RETRY_CONNREFUSED} \
|
||||
--fail \
|
||||
--silent \
|
||||
-H 'Metadata-Flavor: Google' \
|
||||
"http://metadata/computeMetadata/v1/${1}"
|
||||
}
|
||||
|
||||
# A helper function for copying manifests and setting dir/files
|
||||
# permissions.
|
||||
#
|
||||
@ -1974,10 +2167,25 @@ function start-fluentd-resource-update {
|
||||
wait-for-apiserver-and-update-fluentd &
|
||||
}
|
||||
|
||||
# Update {{ container-runtime }} with actual container runtime name.
|
||||
# Update {{ container-runtime }} with actual container runtime name,
|
||||
# and {{ container-runtime-endpoint }} with actual container runtime
|
||||
# endpoint.
|
||||
function update-container-runtime {
|
||||
local -r file="$1"
|
||||
local -r container_runtime_endpoint="${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/dockershim.sock}"
|
||||
sed -i \
|
||||
-e "s@{{ *container_runtime *}}@${CONTAINER_RUNTIME_NAME:-docker}@g" \
|
||||
-e "s@{{ *container_runtime_endpoint *}}@${container_runtime_endpoint#unix://}@g" \
|
||||
"${file}"
|
||||
}
|
||||
|
||||
# Remove configuration in yaml file if node journal is not enabled.
|
||||
function update-node-journal {
|
||||
local -r configmap_yaml="$1"
|
||||
sed -i -e "s@{{ *container_runtime *}}@${CONTAINER_RUNTIME_NAME:-docker}@g" "${configmap_yaml}"
|
||||
if [[ "${ENABLE_NODE_JOURNAL:-}" != "true" ]]; then
|
||||
# Removes all lines between two patterns (throws away node-journal)
|
||||
sed -i -e "/# BEGIN_NODE_JOURNAL/,/# END_NODE_JOURNAL/d" "${configmap_yaml}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Updates parameters in yaml file for prometheus-to-sd configuration, or
|
||||
@ -1994,23 +2202,60 @@ function update-prometheus-to-sd-parameters {
|
||||
|
||||
# Updates parameters in yaml file for event-exporter configuration
|
||||
function update-event-exporter {
|
||||
sed -i -e "s@{{ *event_exporter_zone *}}@${ZONE:-}@g" "$1"
|
||||
local -r stackdriver_resource_model="${LOGGING_STACKDRIVER_RESOURCE_TYPES:-old}"
|
||||
sed -i -e "s@{{ exporter_sd_resource_model }}@${stackdriver_resource_model}@g" "$1"
|
||||
}
|
||||
|
||||
function update-dashboard-controller {
|
||||
if [ -n "${CUSTOM_KUBE_DASHBOARD_BANNER:-}" ]; then
|
||||
sed -i -e "s@\( \+\)# PLATFORM-SPECIFIC ARGS HERE@\1- --system-banner=${CUSTOM_KUBE_DASHBOARD_BANNER}\n\1- --system-banner-severity=WARNING@" "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Sets up the manifests of coreDNS for k8s addons.
|
||||
function setup-coredns-manifest {
|
||||
local -r coredns_file="${dst_dir}/dns/coredns.yaml"
|
||||
mv "${dst_dir}/dns/coredns.yaml.in" "${coredns_file}"
|
||||
local -r coredns_file="${dst_dir}/dns/coredns/coredns.yaml"
|
||||
mv "${dst_dir}/dns/coredns/coredns.yaml.in" "${coredns_file}"
|
||||
# Replace the salt configurations with variable values.
|
||||
sed -i -e "s@{{ *pillar\['dns_domain'\] *}}@${DNS_DOMAIN}@g" "${coredns_file}"
|
||||
sed -i -e "s@{{ *pillar\['dns_server'\] *}}@${DNS_SERVER_IP}@g" "${coredns_file}"
|
||||
sed -i -e "s@{{ *pillar\['service_cluster_ip_range'\] *}}@${SERVICE_CLUSTER_IP_RANGE}@g" "${coredns_file}"
|
||||
|
||||
if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "dns-horizontal-autoscaler" "gce"
|
||||
local -r dns_autoscaler_file="${dst_dir}/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml"
|
||||
sed -i'' -e "s@{{.Target}}@${COREDNS_AUTOSCALER}@g" "${dns_autoscaler_file}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Sets up the manifests of Fluentd configmap and yamls for k8s addons.
|
||||
function setup-fluentd {
|
||||
local -r dst_dir="$1"
|
||||
local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml"
|
||||
# Ingest logs against new resources like "k8s_container" and "k8s_node" if
|
||||
# LOGGING_STACKDRIVER_RESOURCE_TYPES is "new".
|
||||
# Ingest logs against old resources like "gke_container" and "gce_instance" if
|
||||
# LOGGING_STACKDRIVER_RESOURCE_TYPES is "old".
|
||||
if [[ "${LOGGING_STACKDRIVER_RESOURCE_TYPES:-old}" == "new" ]]; then
|
||||
local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap.yaml"
|
||||
fluentd_gcp_configmap_name="fluentd-gcp-config"
|
||||
else
|
||||
local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap-old.yaml"
|
||||
fluentd_gcp_configmap_name="fluentd-gcp-config-old"
|
||||
fi
|
||||
sed -i -e "s@{{ fluentd_gcp_configmap_name }}@${fluentd_gcp_configmap_name}@g" "${fluentd_gcp_yaml}"
|
||||
fluentd_gcp_version="${FLUENTD_GCP_VERSION:-0.2-1.5.30-1-k8s}"
|
||||
sed -i -e "s@{{ fluentd_gcp_version }}@${fluentd_gcp_version}@g" "${fluentd_gcp_yaml}"
|
||||
update-prometheus-to-sd-parameters ${fluentd_gcp_yaml}
|
||||
start-fluentd-resource-update ${fluentd_gcp_yaml}
|
||||
update-container-runtime ${fluentd_gcp_configmap_yaml}
|
||||
update-node-journal ${fluentd_gcp_configmap_yaml}
|
||||
}
|
||||
|
||||
# Sets up the manifests of kube-dns for k8s addons.
|
||||
function setup-kube-dns-manifest {
|
||||
local -r kubedns_file="${dst_dir}/dns/kube-dns.yaml"
|
||||
mv "${dst_dir}/dns/kube-dns.yaml.in" "${kubedns_file}"
|
||||
local -r kubedns_file="${dst_dir}/dns/kube-dns/kube-dns.yaml"
|
||||
mv "${dst_dir}/dns/kube-dns/kube-dns.yaml.in" "${kubedns_file}"
|
||||
if [ -n "${CUSTOM_KUBE_DNS_YAML:-}" ]; then
|
||||
# Replace with custom GKE kube-dns deployment.
|
||||
cat > "${kubedns_file}" <<EOF
|
||||
@ -2024,6 +2269,38 @@ EOF
|
||||
|
||||
if [[ "${ENABLE_DNS_HORIZONTAL_AUTOSCALER:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "dns-horizontal-autoscaler" "gce"
|
||||
local -r dns_autoscaler_file="${dst_dir}/dns-horizontal-autoscaler/dns-horizontal-autoscaler.yaml"
|
||||
sed -i'' -e "s@{{.Target}}@${KUBEDNS_AUTOSCALER}@g" "${dns_autoscaler_file}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Sets up the manifests of netd for k8s addons.
|
||||
function setup-netd-manifest {
|
||||
local -r netd_file="${dst_dir}/netd/netd.yaml"
|
||||
mkdir -p "${dst_dir}/netd"
|
||||
touch "${netd_file}"
|
||||
if [ -n "${CUSTOM_NETD_YAML:-}" ]; then
|
||||
# Replace with custom GCP netd deployment.
|
||||
cat > "${netd_file}" <<EOF
|
||||
$(echo "$CUSTOM_NETD_YAML")
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
# A helper function to set up a custom yaml for a k8s addon.
|
||||
#
|
||||
# $1: addon category under /etc/kubernetes
|
||||
# $2: manifest source dir
|
||||
# $3: manifest file
|
||||
# $4: custom yaml
|
||||
function setup-addon-custom-yaml {
|
||||
local -r manifest_path="/etc/kubernetes/$1/$2/$3"
|
||||
local -r custom_yaml="$4"
|
||||
if [ -n "${custom_yaml:-}" ]; then
|
||||
# Replace with custom manifest.
|
||||
cat > "${manifest_path}" <<EOF
|
||||
$custom_yaml
|
||||
EOF
|
||||
fi
|
||||
}
|
||||
|
||||
@ -2060,6 +2337,11 @@ EOF
|
||||
prepare-kube-proxy-manifest-variables "$src_dir/kube-proxy/kube-proxy-ds.yaml"
|
||||
setup-addon-manifests "addons" "kube-proxy"
|
||||
fi
|
||||
# Setup prometheus stack for monitoring kubernetes cluster
|
||||
if [[ "${ENABLE_PROMETHEUS_MONITORING:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "prometheus"
|
||||
fi
|
||||
# Setup cluster monitoring using heapster
|
||||
if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "influxdb" ]] || \
|
||||
[[ "${ENABLE_CLUSTER_MONITORING:-}" == "google" ]] || \
|
||||
[[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] || \
|
||||
@ -2089,6 +2371,7 @@ EOF
|
||||
fi
|
||||
|
||||
sed -i -e "s@{{ cluster_name }}@${CLUSTER_NAME}@g" "${controller_yaml}"
|
||||
sed -i -e "s@{{ cluster_location }}@${ZONE}@g" "${controller_yaml}"
|
||||
sed -i -e "s@{{ *base_metrics_memory *}}@${base_metrics_memory}@g" "${controller_yaml}"
|
||||
sed -i -e "s@{{ *base_metrics_cpu *}}@${base_metrics_cpu}@g" "${controller_yaml}"
|
||||
sed -i -e "s@{{ *base_eventer_memory *}}@${base_eventer_memory}@g" "${controller_yaml}"
|
||||
@ -2108,15 +2391,17 @@ EOF
|
||||
if [[ "${ENABLE_CLUSTER_MONITORING:-}" == "stackdriver" ]] ||
|
||||
([[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]] &&
|
||||
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]); then
|
||||
if [[ "${ENABLE_METADATA_AGENT:-}" == "stackdriver" ]] &&
|
||||
[[ "${METADATA_AGENT_VERSION:-}" != "" ]]; then
|
||||
if [[ "${ENABLE_METADATA_AGENT:-}" == "stackdriver" ]]; then
|
||||
metadata_agent_cpu_request="${METADATA_AGENT_CPU_REQUEST:-40m}"
|
||||
metadata_agent_memory_request="${METADATA_AGENT_MEMORY_REQUEST:-50Mi}"
|
||||
metadata_agent_cluster_level_cpu_request="${METADATA_AGENT_CLUSTER_LEVEL_CPU_REQUEST:-40m}"
|
||||
metadata_agent_cluster_level_memory_request="${METADATA_AGENT_CLUSTER_LEVEL_MEMORY_REQUEST:-50Mi}"
|
||||
setup-addon-manifests "addons" "metadata-agent/stackdriver"
|
||||
daemon_set_yaml="${dst_dir}/metadata-agent/stackdriver/metadata-agent.yaml"
|
||||
sed -i -e "s@{{ metadata_agent_version }}@${METADATA_AGENT_VERSION}@g" "${daemon_set_yaml}"
|
||||
sed -i -e "s@{{ metadata_agent_cpu_request }}@${metadata_agent_cpu_request}@g" "${daemon_set_yaml}"
|
||||
sed -i -e "s@{{ metadata_agent_memory_request }}@${metadata_agent_memory_request}@g" "${daemon_set_yaml}"
|
||||
metadata_agent_yaml="${dst_dir}/metadata-agent/stackdriver/metadata-agent.yaml"
|
||||
sed -i -e "s@{{ metadata_agent_cpu_request }}@${metadata_agent_cpu_request}@g" "${metadata_agent_yaml}"
|
||||
sed -i -e "s@{{ metadata_agent_memory_request }}@${metadata_agent_memory_request}@g" "${metadata_agent_yaml}"
|
||||
sed -i -e "s@{{ metadata_agent_cluster_level_cpu_request }}@${metadata_agent_cluster_level_cpu_request}@g" "${metadata_agent_yaml}"
|
||||
sed -i -e "s@{{ metadata_agent_cluster_level_memory_request }}@${metadata_agent_cluster_level_memory_request}@g" "${metadata_agent_yaml}"
|
||||
fi
|
||||
fi
|
||||
if [[ "${ENABLE_METRICS_SERVER:-}" == "true" ]]; then
|
||||
@ -2126,13 +2411,17 @@ EOF
|
||||
setup-addon-manifests "addons" "device-plugins/nvidia-gpu"
|
||||
fi
|
||||
if [[ "${ENABLE_CLUSTER_DNS:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "dns"
|
||||
if [[ "${CLUSTER_DNS_CORE_DNS:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "dns/coredns"
|
||||
setup-coredns-manifest
|
||||
else
|
||||
setup-addon-manifests "addons" "dns/kube-dns"
|
||||
setup-kube-dns-manifest
|
||||
fi
|
||||
fi
|
||||
if [[ "${ENABLE_NETD:-}" == "true" ]]; then
|
||||
setup-netd-manifest
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
||||
[[ "${LOGGING_DESTINATION:-}" == "elasticsearch" ]] && \
|
||||
[[ "${ENABLE_CLUSTER_LOGGING:-}" == "true" ]]; then
|
||||
@ -2143,19 +2432,15 @@ EOF
|
||||
if [[ "${ENABLE_NODE_LOGGING:-}" == "true" ]] && \
|
||||
[[ "${LOGGING_DESTINATION:-}" == "gcp" ]]; then
|
||||
setup-addon-manifests "addons" "fluentd-gcp"
|
||||
setup-fluentd ${dst_dir}
|
||||
local -r event_exporter_yaml="${dst_dir}/fluentd-gcp/event-exporter.yaml"
|
||||
local -r fluentd_gcp_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-ds.yaml"
|
||||
local -r fluentd_gcp_configmap_yaml="${dst_dir}/fluentd-gcp/fluentd-gcp-configmap.yaml"
|
||||
update-event-exporter ${event_exporter_yaml}
|
||||
fluentd_gcp_version="${FLUENTD_GCP_VERSION:-0.2-1.5.28-1}"
|
||||
sed -i -e "s@{{ fluentd_gcp_version }}@${fluentd_gcp_version}@g" "${fluentd_gcp_yaml}"
|
||||
update-prometheus-to-sd-parameters ${event_exporter_yaml}
|
||||
update-prometheus-to-sd-parameters ${fluentd_gcp_yaml}
|
||||
start-fluentd-resource-update ${fluentd_gcp_yaml}
|
||||
update-container-runtime ${fluentd_gcp_configmap_yaml}
|
||||
fi
|
||||
if [[ "${ENABLE_CLUSTER_UI:-}" == "true" ]]; then
|
||||
setup-addon-manifests "addons" "dashboard"
|
||||
local -r dashboard_controller_yaml="${dst_dir}/dashboard/dashboard-controller.yaml"
|
||||
update-dashboard-controller ${dashboard_controller_yaml}
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "daemonset" ]]; then
|
||||
setup-addon-manifests "addons" "node-problem-detector"
|
||||
@ -2170,6 +2455,9 @@ EOF
|
||||
if [[ "${NETWORK_POLICY_PROVIDER:-}" == "calico" ]]; then
|
||||
setup-addon-manifests "addons" "calico-policy-controller"
|
||||
|
||||
setup-addon-custom-yaml "addons" "calico-policy-controller" "calico-node-daemonset.yaml" "${CUSTOM_CALICO_NODE_DAEMONSET_YAML:-}"
|
||||
setup-addon-custom-yaml "addons" "calico-policy-controller" "typha-deployment.yaml" "${CUSTOM_TYPHA_DEPLOYMENT_YAML:-}"
|
||||
|
||||
# Configure Calico CNI directory.
|
||||
local -r ds_file="${dst_dir}/calico-policy-controller/calico-node-daemonset.yaml"
|
||||
sed -i -e "s@__CALICO_CNI_DIR__@/home/kubernetes/bin@g" "${ds_file}"
|
||||
@ -2204,8 +2492,9 @@ EOF
|
||||
# Starts an image-puller - used in test clusters.
|
||||
function start-image-puller {
|
||||
echo "Start image-puller"
|
||||
cp "${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest" \
|
||||
/etc/kubernetes/manifests/
|
||||
local -r e2e_image_puller_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/e2e-image-puller.manifest"
|
||||
update-container-runtime "${e2e_image_puller_manifest}"
|
||||
cp "${e2e_image_puller_manifest}" /etc/kubernetes/manifests/
|
||||
}
|
||||
|
||||
# Setups manifests for ingress controller and gce-specific policies for service controller.
|
||||
@ -2218,11 +2507,19 @@ function start-lb-controller {
|
||||
prepare-log-file /var/log/glbc.log
|
||||
setup-addon-manifests "addons" "cluster-loadbalancing/glbc"
|
||||
|
||||
local -r glbc_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
|
||||
if [[ ! -z "${GCE_GLBC_IMAGE:-}" ]]; then
|
||||
sed -i "s@image:.*@image: ${GCE_GLBC_IMAGE}@" "${glbc_manifest}"
|
||||
local -r src_manifest="${KUBE_HOME}/kube-manifests/kubernetes/gci-trusty/glbc.manifest"
|
||||
local -r dest_manifest="/etc/kubernetes/manifests/glbc.manifest"
|
||||
|
||||
if [[ -n "${CUSTOM_INGRESS_YAML:-}" ]]; then
|
||||
echo "${CUSTOM_INGRESS_YAML}" > "${dest_manifest}"
|
||||
else
|
||||
cp "${src_manifest}" "${dest_manifest}"
|
||||
fi
|
||||
|
||||
# Override the glbc image if GCE_GLBC_IMAGE is specified.
|
||||
if [[ -n "${GCE_GLBC_IMAGE:-}" ]]; then
|
||||
sed -i "s|image:.*|image: ${GCE_GLBC_IMAGE}|" "${dest_manifest}"
|
||||
fi
|
||||
cp "${glbc_manifest}" /etc/kubernetes/manifests/
|
||||
fi
|
||||
}
|
||||
|
||||
@ -2243,6 +2540,15 @@ function setup-kubelet-dir {
|
||||
mount -B -o remount,exec,suid,dev /var/lib/kubelet
|
||||
}
|
||||
|
||||
# Override for GKE custom master setup scripts (no-op outside of GKE).
|
||||
function gke-master-start {
|
||||
if [[ -e "${KUBE_HOME}/bin/gke-internal-configure-helper.sh" ]]; then
|
||||
echo "Running GKE internal configuration script"
|
||||
. "${KUBE_HOME}/bin/gke-internal-configure-helper.sh"
|
||||
gke-internal-master-start
|
||||
fi
|
||||
}
|
||||
|
||||
function reset-motd {
|
||||
# kubelet is installed both on the master and nodes, and the version is easy to parse (unlike kubectl)
|
||||
local -r version="$("${KUBE_HOME}"/bin/kubelet --version=true | cut -f2 -d " ")"
|
||||
@ -2282,6 +2588,16 @@ EOF
|
||||
function override-kubectl {
|
||||
echo "overriding kubectl"
|
||||
echo "export PATH=${KUBE_HOME}/bin:\$PATH" > /etc/profile.d/kube_env.sh
|
||||
# Add ${KUBE_HOME}/bin into sudoer secure path.
|
||||
local sudo_path
|
||||
sudo_path=$(sudo env | grep "^PATH=")
|
||||
if [[ -n "${sudo_path}" ]]; then
|
||||
sudo_path=${sudo_path#PATH=}
|
||||
(
|
||||
umask 027
|
||||
echo "Defaults secure_path=\"${KUBE_HOME}/bin:${sudo_path}\"" > /etc/sudoers.d/kube_secure_path
|
||||
)
|
||||
fi
|
||||
}
|
||||
|
||||
function override-pv-recycler {
|
||||
@ -2319,89 +2635,106 @@ EOF
|
||||
}
|
||||
|
||||
########### Main Function ###########
|
||||
echo "Start to configure instance for kubernetes"
|
||||
function main() {
|
||||
echo "Start to configure instance for kubernetes"
|
||||
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
CONTAINERIZED_MOUNTER_HOME="${KUBE_HOME}/containerized_mounter"
|
||||
PV_RECYCLER_OVERRIDE_TEMPLATE="${KUBE_HOME}/kube-manifests/kubernetes/pv-recycler-template.yaml"
|
||||
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
if [[ ! -e "${KUBE_HOME}/kube-env" ]]; then
|
||||
echo "The ${KUBE_HOME}/kube-env file does not exist!! Terminate cluster initialization."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager and scheduler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
|
||||
if [[ -f "${KUBE_HOME}/kubelet-config.yaml" ]]; then
|
||||
echo "Found Kubelet config file at ${KUBE_HOME}/kubelet-config.yaml"
|
||||
KUBELET_CONFIG_FILE_ARG="--config ${KUBE_HOME}/kubelet-config.yaml"
|
||||
fi
|
||||
|
||||
if [[ -e "${KUBE_HOME}/kube-master-certs" ]]; then
|
||||
source "${KUBE_HOME}/kube-master-certs"
|
||||
fi
|
||||
|
||||
if [[ -n "${KUBE_USER:-}" ]]; then
|
||||
if ! [[ "${KUBE_USER}" =~ ^[-._@a-zA-Z0-9]+$ ]]; then
|
||||
echo "Bad KUBE_USER format."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# generate the controller manager, scheduler and cluster autoscaler tokens here since they are only used on the master.
|
||||
KUBE_CONTROLLER_MANAGER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_SCHEDULER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
KUBE_CLUSTER_AUTOSCALER_TOKEN=$(dd if=/dev/urandom bs=128 count=1 2>/dev/null | base64 | tr -d "=+/" | dd bs=32 count=1 2>/dev/null)
|
||||
|
||||
setup-os-params
|
||||
config-ip-firewall
|
||||
create-dirs
|
||||
setup-kubelet-dir
|
||||
ensure-local-ssds
|
||||
setup-logrotate
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
mount-master-pd
|
||||
create-node-pki
|
||||
create-master-pki
|
||||
create-master-auth
|
||||
create-master-kubelet-auth
|
||||
create-master-etcd-auth
|
||||
override-pv-recycler
|
||||
gke-master-start
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
}
|
||||
|
||||
# use --source-only to test functions defined in this script.
|
||||
if [[ "$#" -eq 1 && "${1}" == "--source-only" ]]; then
|
||||
:
|
||||
else
|
||||
create-node-pki
|
||||
create-kubelet-kubeconfig ${KUBERNETES_MASTER_NAME}
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
create-kubeproxy-user-kubeconfig
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
create-node-problem-detector-kubeconfig
|
||||
fi
|
||||
main "${@}"
|
||||
fi
|
||||
|
||||
override-kubectl
|
||||
# Run the containerized mounter once to pre-cache the container image.
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" == "docker" ]]; then
|
||||
assemble-docker-flags
|
||||
fi
|
||||
start-kubelet
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
compute-master-manifest-variables
|
||||
start-etcd-servers
|
||||
start-etcd-empty-dir-cleanup-pod
|
||||
start-kube-apiserver
|
||||
start-kube-controller-manager
|
||||
start-kube-scheduler
|
||||
start-kube-addons
|
||||
start-cluster-autoscaler
|
||||
start-lb-controller
|
||||
start-rescheduler
|
||||
else
|
||||
if [[ "${KUBE_PROXY_DAEMONSET:-}" != "true" ]]; then
|
||||
start-kube-proxy
|
||||
fi
|
||||
if [[ "${PREPULL_E2E_IMAGES:-}" == "true" ]]; then
|
||||
start-image-puller
|
||||
fi
|
||||
if [[ "${ENABLE_NODE_PROBLEM_DETECTOR:-}" == "standalone" ]]; then
|
||||
start-node-problem-detector
|
||||
fi
|
||||
fi
|
||||
reset-motd
|
||||
prepare-mounter-rootfs
|
||||
modprobe configs
|
||||
echo "Done for the configuration for kubernetes"
|
||||
|
||||
135
vendor/k8s.io/kubernetes/cluster/gce/gci/configure.sh
generated
vendored
135
vendor/k8s.io/kubernetes/cluster/gce/gci/configure.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -25,9 +25,11 @@ set -o pipefail
|
||||
|
||||
### Hardcoded constants
|
||||
DEFAULT_CNI_VERSION="v0.6.0"
|
||||
DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f"
|
||||
DEFAULT_NPD_VERSION="v0.4.1"
|
||||
DEFAULT_NPD_SHA1="a57a3fe64cab8a18ec654f5cef0aec59dae62568"
|
||||
DEFAULT_CNI_SHA1="d595d3ded6499a64e8dac02466e2f5f2ce257c9f"
|
||||
DEFAULT_NPD_VERSION="v0.5.0"
|
||||
DEFAULT_NPD_SHA1="650ecfb2ae495175ee43706d0bd862a1ea7f1395"
|
||||
DEFAULT_CRICTL_VERSION="v1.11.0"
|
||||
DEFAULT_CRICTL_SHA1="8f5142b985d314cdebb51afd55054d5ec00c442a"
|
||||
DEFAULT_MOUNTER_TAR_SHA="8003b798cf33c7f91320cd6ee5cec4fa22244571"
|
||||
###
|
||||
|
||||
@ -54,37 +56,59 @@ EOF
|
||||
|
||||
function download-kube-env {
|
||||
# Fetch kube-env from GCE metadata server.
|
||||
(umask 700;
|
||||
local -r tmp_kube_env="/tmp/kube-env.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_env}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kube_env="/tmp/kube-env.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_env}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-env
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
|
||||
''' < "${tmp_kube_env}" > "${KUBE_HOME}/kube-env")
|
||||
rm -f "${tmp_kube_env}"
|
||||
rm -f "${tmp_kube_env}"
|
||||
)
|
||||
}
|
||||
|
||||
function download-kubelet-config {
|
||||
local -r dest="$1"
|
||||
echo "Downloading Kubelet config file, if it exists"
|
||||
# Fetch kubelet config file from GCE metadata server.
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kubelet_config="/tmp/kubelet-config.yaml"
|
||||
if curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kubelet_config}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kubelet-config; then
|
||||
# only write to the final location if curl succeeds
|
||||
mv "${tmp_kubelet_config}" "${dest}"
|
||||
elif [[ "${REQUIRE_METADATA_KUBELET_CONFIG_FILE:-false}" == "true" ]]; then
|
||||
echo "== Failed to download required Kubelet config file from metadata server =="
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
}
|
||||
|
||||
function download-kube-master-certs {
|
||||
# Fetch kube-env from GCE metadata server.
|
||||
(umask 700;
|
||||
local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_master_certs}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
(
|
||||
umask 077
|
||||
local -r tmp_kube_master_certs="/tmp/kube-master-certs.yaml"
|
||||
curl --fail --retry 5 --retry-delay 3 ${CURL_RETRY_CONNREFUSED} --silent --show-error \
|
||||
-H "X-Google-Metadata-Request: True" \
|
||||
-o "${tmp_kube_master_certs}" \
|
||||
http://metadata.google.internal/computeMetadata/v1/instance/attributes/kube-master-certs
|
||||
# Convert the yaml format file into a shell-style file.
|
||||
eval $(python -c '''
|
||||
import pipes,sys,yaml
|
||||
for k,v in yaml.load(sys.stdin).iteritems():
|
||||
print("readonly {var}={value}".format(var = k, value = pipes.quote(str(v))))
|
||||
''' < "${tmp_kube_master_certs}" > "${KUBE_HOME}/kube-master-certs")
|
||||
rm -f "${tmp_kube_master_certs}"
|
||||
rm -f "${tmp_kube_master_certs}"
|
||||
)
|
||||
}
|
||||
|
||||
@ -175,15 +199,15 @@ function install-node-problem-detector {
|
||||
local -r npd_version="${DEFAULT_NPD_VERSION}"
|
||||
local -r npd_sha1="${DEFAULT_NPD_SHA1}"
|
||||
fi
|
||||
local -r npd_tar="node-problem-detector-${npd_version}.tar.gz"
|
||||
|
||||
if is-preloaded "node-problem-detector" "${npd_sha1}"; then
|
||||
if is-preloaded "${npd_tar}" "${npd_sha1}"; then
|
||||
echo "node-problem-detector is preloaded."
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Downloading node problem detector."
|
||||
local -r npd_release_path="https://storage.googleapis.com/kubernetes-release"
|
||||
local -r npd_tar="node-problem-detector-${npd_version}.tar.gz"
|
||||
download-or-bust "${npd_sha1}" "${npd_release_path}/node-problem-detector/${npd_tar}"
|
||||
local -r npd_dir="${KUBE_HOME}/node-problem-detector"
|
||||
mkdir -p "${npd_dir}"
|
||||
@ -212,6 +236,47 @@ function install-cni-binaries {
|
||||
rm -f "${KUBE_HOME}/${cni_tar}"
|
||||
}
|
||||
|
||||
# Install crictl binary.
|
||||
function install-crictl {
|
||||
if [[ -n "${CRICTL_VERSION:-}" ]]; then
|
||||
local -r crictl_version="${CRICTL_VERSION}"
|
||||
local -r crictl_sha1="${CRICTL_TAR_HASH}"
|
||||
else
|
||||
local -r crictl_version="${DEFAULT_CRICTL_VERSION}"
|
||||
local -r crictl_sha1="${DEFAULT_CRICTL_SHA1}"
|
||||
fi
|
||||
local -r crictl="crictl-${crictl_version}-linux-amd64"
|
||||
|
||||
if is-preloaded "${crictl}" "${crictl_sha1}"; then
|
||||
echo "crictl is preloaded"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "Downloading crictl"
|
||||
local -r crictl_path="https://storage.googleapis.com/kubernetes-release/crictl"
|
||||
download-or-bust "${crictl_sha1}" "${crictl_path}/${crictl}"
|
||||
mv "${KUBE_HOME}/${crictl}" "${KUBE_BIN}/crictl"
|
||||
chmod a+x "${KUBE_BIN}/crictl"
|
||||
|
||||
# Create crictl config file.
|
||||
cat > /etc/crictl.yaml <<EOF
|
||||
runtime-endpoint: ${CONTAINER_RUNTIME_ENDPOINT:-unix:///var/run/dockershim.sock}
|
||||
EOF
|
||||
}
|
||||
|
||||
function install-exec-auth-plugin {
|
||||
if [[ ! "${EXEC_AUTH_PLUGIN_URL:-}" ]]; then
|
||||
return
|
||||
fi
|
||||
local -r plugin_url="${EXEC_AUTH_PLUGIN_URL}"
|
||||
local -r plugin_sha1="${EXEC_AUTH_PLUGIN_SHA1}"
|
||||
|
||||
echo "Downloading gke-exec-auth-plugin binary"
|
||||
download-or-bust "${plugin_sha1}" "${plugin_url}"
|
||||
mv "${KUBE_HOME}/gke-exec-auth-plugin" "${KUBE_BIN}/gke-exec-auth-plugin"
|
||||
chmod a+x "${KUBE_BIN}/gke-exec-auth-plugin"
|
||||
}
|
||||
|
||||
function install-kube-manifests {
|
||||
# Put kube-system pods manifests in ${KUBE_HOME}/kube-manifests/.
|
||||
local dst_dir="${KUBE_HOME}/kube-manifests"
|
||||
@ -242,6 +307,10 @@ function install-kube-manifests {
|
||||
xargs sed -ri "s@(image\":\s+\")k8s.gcr.io@\1${kube_addon_registry}@"
|
||||
fi
|
||||
cp "${dst_dir}/kubernetes/gci-trusty/gci-configure-helper.sh" "${KUBE_BIN}/configure-helper.sh"
|
||||
if [[ -e "${dst_dir}/kubernetes/gci-trusty/gke-internal-configure-helper.sh" ]]; then
|
||||
cp "${dst_dir}/kubernetes/gci-trusty/gke-internal-configure-helper.sh" "${KUBE_BIN}/"
|
||||
fi
|
||||
|
||||
cp "${dst_dir}/kubernetes/gci-trusty/health-monitor.sh" "${KUBE_BIN}/health-monitor.sh"
|
||||
|
||||
rm -f "${KUBE_HOME}/${manifests_tar}"
|
||||
@ -348,6 +417,13 @@ function install-kube-binary-config {
|
||||
remount-flexvolume-directory "${VOLUME_PLUGIN_DIR}"
|
||||
fi
|
||||
|
||||
# Install crictl on each node.
|
||||
install-crictl
|
||||
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "false" ]]; then
|
||||
install-exec-auth-plugin
|
||||
fi
|
||||
|
||||
# Clean up.
|
||||
rm -rf "${KUBE_HOME}/kubernetes"
|
||||
rm -f "${KUBE_HOME}/${server_binary_tar}"
|
||||
@ -356,13 +432,24 @@ function install-kube-binary-config {
|
||||
|
||||
######### Main Function ##########
|
||||
echo "Start to install kubernetes files"
|
||||
# if install fails, message-of-the-day (motd) will warn at login shell
|
||||
set-broken-motd
|
||||
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
KUBE_BIN="${KUBE_HOME}/bin"
|
||||
|
||||
# download and source kube-env
|
||||
download-kube-env
|
||||
source "${KUBE_HOME}/kube-env"
|
||||
|
||||
download-kubelet-config "${KUBE_HOME}/kubelet-config.yaml"
|
||||
|
||||
# master certs
|
||||
if [[ "${KUBERNETES_MASTER:-}" == "true" ]]; then
|
||||
download-kube-master-certs
|
||||
fi
|
||||
|
||||
# binaries and kube-system manifests
|
||||
install-kube-binary-config
|
||||
|
||||
echo "Done for installing kubernetes files"
|
||||
|
||||
172
vendor/k8s.io/kubernetes/cluster/gce/gci/configure_helper_test.go
generated
vendored
Normal file
172
vendor/k8s.io/kubernetes/cluster/gce/gci/configure_helper_test.go
generated
vendored
Normal file
@ -0,0 +1,172 @@
|
||||
/*
|
||||
Copyright 2017 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package gci
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"text/template"
|
||||
|
||||
"k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/kubernetes/pkg/api/legacyscheme"
|
||||
)
|
||||
|
||||
const (
|
||||
envScriptFileName = "kube-env"
|
||||
configureHelperScriptName = "configure-helper.sh"
|
||||
)
|
||||
|
||||
type ManifestTestCase struct {
|
||||
pod v1.Pod
|
||||
envScriptPath string
|
||||
manifest string
|
||||
auxManifests []string
|
||||
kubeHome string
|
||||
manifestSources string
|
||||
manifestDestination string
|
||||
manifestTemplateDir string
|
||||
manifestTemplate string
|
||||
manifestFuncName string
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func newManifestTestCase(t *testing.T, manifest, funcName string, auxManifests []string) *ManifestTestCase {
|
||||
c := &ManifestTestCase{
|
||||
t: t,
|
||||
manifest: manifest,
|
||||
auxManifests: auxManifests,
|
||||
manifestFuncName: funcName,
|
||||
}
|
||||
|
||||
d, err := ioutil.TempDir("", "configure-helper-test")
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
|
||||
c.kubeHome = d
|
||||
c.envScriptPath = filepath.Join(c.kubeHome, envScriptFileName)
|
||||
c.manifestSources = filepath.Join(c.kubeHome, "kube-manifests", "kubernetes", "gci-trusty")
|
||||
|
||||
currentPath, err := os.Getwd()
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to get current directory: %v", err)
|
||||
}
|
||||
gceDir := filepath.Dir(currentPath)
|
||||
c.manifestTemplateDir = filepath.Join(gceDir, "manifests")
|
||||
c.manifestTemplate = filepath.Join(c.manifestTemplateDir, c.manifest)
|
||||
c.manifestDestination = filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests", c.manifest)
|
||||
|
||||
c.mustCopyFromTemplate()
|
||||
c.mustCopyAuxFromTemplate()
|
||||
c.mustCreateManifestDstDir()
|
||||
|
||||
return c
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyFromTemplate() {
|
||||
if err := os.MkdirAll(c.manifestSources, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create source directory: %v", err)
|
||||
}
|
||||
|
||||
if err := copyFile(c.manifestTemplate, filepath.Join(c.manifestSources, c.manifest)); err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest to KUBE_HOME: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCopyAuxFromTemplate() {
|
||||
for _, m := range c.auxManifests {
|
||||
err := copyFile(filepath.Join(c.manifestTemplateDir, m), filepath.Join(c.manifestSources, m))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to copy source manifest %s to KUBE_HOME: %v", m, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateManifestDstDir() {
|
||||
p := filepath.Join(filepath.Join(c.kubeHome, "etc", "kubernetes", "manifests"))
|
||||
if err := os.MkdirAll(p, os.ModePerm); err != nil {
|
||||
c.t.Fatalf("Failed to create designation folder for kube-apiserver.manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustCreateEnv(envTemplate string, env interface{}) {
|
||||
f, err := os.Create(filepath.Join(c.kubeHome, envScriptFileName))
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to create envScript: %v", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
t := template.Must(template.New("env").Parse(envTemplate))
|
||||
|
||||
if err = t.Execute(f, env); err != nil {
|
||||
c.t.Fatalf("Failed to execute template: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustInvokeFunc(envTemplate string, env interface{}) {
|
||||
c.mustCreateEnv(envTemplate, env)
|
||||
args := fmt.Sprintf("source %s ; source %s --source-only ; %s", c.envScriptPath, configureHelperScriptName, c.manifestFuncName)
|
||||
cmd := exec.Command("bash", "-c", args)
|
||||
|
||||
bs, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
c.t.Logf("%s", bs)
|
||||
c.t.Fatalf("Failed to run configure-helper.sh: %v", err)
|
||||
}
|
||||
c.t.Logf("%s", string(bs))
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) mustLoadPodFromManifest() {
|
||||
json, err := ioutil.ReadFile(c.manifestDestination)
|
||||
if err != nil {
|
||||
c.t.Fatalf("Failed to read manifest: %s, %v", c.manifestDestination, err)
|
||||
}
|
||||
|
||||
if err := runtime.DecodeInto(legacyscheme.Codecs.UniversalDecoder(), json, &c.pod); err != nil {
|
||||
c.t.Fatalf("Failed to decode manifest: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ManifestTestCase) tearDown() {
|
||||
os.RemoveAll(c.kubeHome)
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) (err error) {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
cerr := out.Close()
|
||||
if cerr == nil {
|
||||
err = cerr
|
||||
}
|
||||
}()
|
||||
_, err = io.Copy(out, in)
|
||||
return err
|
||||
}
|
||||
8
vendor/k8s.io/kubernetes/cluster/gce/gci/flexvolume_node_setup.sh
generated
vendored
8
vendor/k8s.io/kubernetes/cluster/gce/gci/flexvolume_node_setup.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2017 The Kubernetes Authors.
|
||||
#
|
||||
@ -81,10 +81,10 @@ flex_clean() {
|
||||
umount_silent ${MOUNTER_PATH}
|
||||
rm -rf ${MOUNTER_PATH}
|
||||
|
||||
if [ -n ${IMAGE_URL:-} ]; then
|
||||
if [[ -n ${IMAGE_URL:-} ]]; then
|
||||
docker rmi -f ${IMAGE_URL} &> /dev/null || /bin/true
|
||||
fi
|
||||
if [ -n ${MOUNTER_DEFAULT_NAME:-} ]; then
|
||||
if [[ -n ${MOUNTER_DEFAULT_NAME:-} ]]; then
|
||||
docker rm -f ${MOUNTER_DEFAULT_NAME} &> /dev/null || /bin/true
|
||||
fi
|
||||
}
|
||||
@ -119,7 +119,7 @@ generate_chroot_wrapper() {
|
||||
|
||||
mkdir -p $wrapper_dir
|
||||
cat >$wrapper_path <<EOF
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
chroot ${MOUNTER_PATH} ${driver_path} "\$@"
|
||||
EOF
|
||||
|
||||
|
||||
51
vendor/k8s.io/kubernetes/cluster/gce/gci/health-monitor.sh
generated
vendored
51
vendor/k8s.io/kubernetes/cluster/gce/gci/health-monitor.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -24,11 +24,39 @@ set -o pipefail
|
||||
|
||||
# We simply kill the process when there is a failure. Another systemd service will
|
||||
# automatically restart the process.
|
||||
function docker_monitoring {
|
||||
while [ 1 ]; do
|
||||
if ! timeout 60 docker ps > /dev/null; then
|
||||
echo "Docker daemon failed!"
|
||||
pkill docker
|
||||
function container_runtime_monitoring {
|
||||
local -r max_attempts=5
|
||||
local attempt=1
|
||||
local -r crictl="${KUBE_HOME}/bin/crictl"
|
||||
local -r container_runtime_name="${CONTAINER_RUNTIME_NAME:-docker}"
|
||||
# We still need to use `docker ps` when container runtime is "docker". This is because
|
||||
# dockershim is still part of kubelet today. When kubelet is down, crictl pods
|
||||
# will also fail, and docker will be killed. This is undesirable especially when
|
||||
# docker live restore is disabled.
|
||||
local healthcheck_command="docker ps"
|
||||
if [[ "${CONTAINER_RUNTIME:-docker}" != "docker" ]]; then
|
||||
healthcheck_command="${crictl} pods"
|
||||
fi
|
||||
# Container runtime startup takes time. Make initial attempts before starting
|
||||
# killing the container runtime.
|
||||
until timeout 60 ${healthcheck_command} > /dev/null; do
|
||||
if (( attempt == max_attempts )); then
|
||||
echo "Max attempt ${max_attempts} reached! Proceeding to monitor container runtime healthiness."
|
||||
break
|
||||
fi
|
||||
echo "$attempt initial attempt \"${healthcheck_command}\"! Trying again in $attempt seconds..."
|
||||
sleep "$(( 2 ** attempt++ ))"
|
||||
done
|
||||
while true; do
|
||||
if ! timeout 60 ${healthcheck_command} > /dev/null; then
|
||||
echo "Container runtime ${container_runtime_name} failed!"
|
||||
if [[ "$container_runtime_name" == "docker" ]]; then
|
||||
# Dump stack of docker daemon for investigation.
|
||||
# Log fle name looks like goroutine-stacks-TIMESTAMP and will be saved to
|
||||
# the exec root directory, which is /var/run/docker/ on Ubuntu and COS.
|
||||
pkill -SIGUSR1 dockerd
|
||||
fi
|
||||
systemctl kill --kill-who=main "${container_runtime_name}"
|
||||
# Wait for a while, as we don't want to kill it again before it is really up.
|
||||
sleep 120
|
||||
else
|
||||
@ -48,7 +76,7 @@ function kubelet_monitoring {
|
||||
# Print the response and/or errors.
|
||||
echo $output
|
||||
echo "Kubelet is unhealthy!"
|
||||
pkill kubelet
|
||||
systemctl kill kubelet
|
||||
# Wait for a while, as we don't want to kill it again before it is really up.
|
||||
sleep 60
|
||||
else
|
||||
@ -60,11 +88,12 @@ function kubelet_monitoring {
|
||||
|
||||
############## Main Function ################
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "Usage: health-monitor.sh <docker/kubelet>"
|
||||
echo "Usage: health-monitor.sh <container-runtime/kubelet>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
KUBE_ENV="/home/kubernetes/kube-env"
|
||||
KUBE_HOME="/home/kubernetes"
|
||||
KUBE_ENV="${KUBE_HOME}/kube-env"
|
||||
if [[ ! -e "${KUBE_ENV}" ]]; then
|
||||
echo "The ${KUBE_ENV} file does not exist!! Terminate health monitoring"
|
||||
exit 1
|
||||
@ -74,8 +103,8 @@ SLEEP_SECONDS=10
|
||||
component=$1
|
||||
echo "Start kubernetes health monitoring for ${component}"
|
||||
source "${KUBE_ENV}"
|
||||
if [[ "${component}" == "docker" ]]; then
|
||||
docker_monitoring
|
||||
if [[ "${component}" == "container-runtime" ]]; then
|
||||
container_runtime_monitoring
|
||||
elif [[ "${component}" == "kubelet" ]]; then
|
||||
kubelet_monitoring
|
||||
else
|
||||
|
||||
2
vendor/k8s.io/kubernetes/cluster/gce/gci/helper.sh
generated
vendored
2
vendor/k8s.io/kubernetes/cluster/gce/gci/helper.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
|
||||
5
vendor/k8s.io/kubernetes/cluster/gce/gci/master-helper.sh
generated
vendored
5
vendor/k8s.io/kubernetes/cluster/gce/gci/master-helper.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -68,6 +68,7 @@ function replicate-master-instance() {
|
||||
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-ensure-gke-docker > "${KUBE_TEMP}/gci-ensure-gke-docker.txt"
|
||||
get-metadata "${existing_master_zone}" "${existing_master_name}" gci-docker-version > "${KUBE_TEMP}/gci-docker-version.txt"
|
||||
get-metadata "${existing_master_zone}" "${existing_master_name}" kube-master-certs > "${KUBE_TEMP}/kube-master-certs.yaml"
|
||||
get-metadata "${existing_master_zone}" "${existing_master_name}" cluster-location > "${KUBE_TEMP}/cluster-location.txt"
|
||||
|
||||
create-master-instance-internal "${REPLICA_NAME}"
|
||||
}
|
||||
@ -106,6 +107,7 @@ function create-master-instance-internal() {
|
||||
"${address:-}" "${enable_ip_aliases:-}" "${IP_ALIAS_SIZE:-}")
|
||||
|
||||
local metadata="kube-env=${KUBE_TEMP}/master-kube-env.yaml"
|
||||
metadata="${metadata},kubelet-config=${KUBE_TEMP}/master-kubelet-config.yaml"
|
||||
metadata="${metadata},user-data=${KUBE_ROOT}/cluster/gce/gci/master.yaml"
|
||||
metadata="${metadata},configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh"
|
||||
metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt"
|
||||
@ -114,6 +116,7 @@ function create-master-instance-internal() {
|
||||
metadata="${metadata},gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt"
|
||||
metadata="${metadata},gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt"
|
||||
metadata="${metadata},kube-master-certs=${KUBE_TEMP}/kube-master-certs.yaml"
|
||||
metadata="${metadata},cluster-location=${KUBE_TEMP}/cluster-location.txt"
|
||||
metadata="${metadata},${MASTER_EXTRA_METADATA}"
|
||||
|
||||
local disk="name=${master_name}-pd"
|
||||
|
||||
8
vendor/k8s.io/kubernetes/cluster/gce/gci/master.yaml
generated
vendored
8
vendor/k8s.io/kubernetes/cluster/gce/gci/master.yaml
generated
vendored
@ -40,12 +40,12 @@ write_files:
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
|
||||
- path: /etc/systemd/system/kube-docker-monitor.service
|
||||
- path: /etc/systemd/system/kube-container-runtime-monitor.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Kubernetes health monitoring for docker
|
||||
Description=Kubernetes health monitoring for container runtime
|
||||
After=kube-master-configuration.service
|
||||
|
||||
[Service]
|
||||
@ -54,7 +54,7 @@ write_files:
|
||||
RemainAfterExit=yes
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
|
||||
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
|
||||
ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime
|
||||
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
@ -120,7 +120,7 @@ runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable kube-master-installation.service
|
||||
- systemctl enable kube-master-configuration.service
|
||||
- systemctl enable kube-docker-monitor.service
|
||||
- systemctl enable kube-container-runtime-monitor.service
|
||||
- systemctl enable kubelet-monitor.service
|
||||
- systemctl enable kube-logrotate.timer
|
||||
- systemctl enable kube-logrotate.service
|
||||
|
||||
21
vendor/k8s.io/kubernetes/cluster/gce/gci/mounter/stage-upload.sh
generated
vendored
21
vendor/k8s.io/kubernetes/cluster/gce/gci/mounter/stage-upload.sh
generated
vendored
@ -23,12 +23,10 @@ set -o errexit
|
||||
set -o pipefail
|
||||
set -o nounset
|
||||
|
||||
RKT_VERSION="v1.18.0"
|
||||
DOCKER2ACI_VERSION="v0.13.0"
|
||||
MOUNTER_VERSION=$1
|
||||
DOCKER_IMAGE=docker://$2
|
||||
MOUNTER_ACI_IMAGE=gci-mounter-${MOUNTER_VERSION}.aci
|
||||
RKT_GCS_DIR=gs://kubernetes-release/rkt/
|
||||
MOUNTER_GCS_DIR=gs://kubernetes-release/gci-mounter/
|
||||
|
||||
TMPDIR=/tmp
|
||||
@ -37,7 +35,6 @@ DOWNLOAD_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-build.XXXXXXXXXX)
|
||||
|
||||
# Setup a staging directory
|
||||
STAGING_DIR=$(mktemp --tmpdir=${TMPDIR} -d gci-mounter-staging.XXXXXXXXXX)
|
||||
RKT_DIR=${STAGING_DIR}/${RKT_VERSION}
|
||||
ACI_DIR=${STAGING_DIR}/gci-mounter
|
||||
CWD=${PWD}
|
||||
|
||||
@ -51,20 +48,8 @@ function cleanup {
|
||||
# Delete temporary directories on exit
|
||||
trap cleanup EXIT
|
||||
|
||||
mkdir ${RKT_DIR}
|
||||
mkdir ${ACI_DIR}
|
||||
|
||||
# Download rkt
|
||||
cd ${DOWNLOAD_DIR}
|
||||
echo "Downloading rkt ${RKT_VERSION}"
|
||||
wget "https://github.com/coreos/rkt/releases/download/${RKT_VERSION}/rkt-${RKT_VERSION}.tar.gz" &> /dev/null
|
||||
echo "Extracting rkt ${RKT_VERSION}"
|
||||
tar xzf rkt-${RKT_VERSION}.tar.gz
|
||||
|
||||
# Stage rkt into working directory
|
||||
cp rkt-${RKT_VERSION}/rkt ${RKT_DIR}/rkt
|
||||
cp rkt-${RKT_VERSION}/stage1-fly.aci ${RKT_DIR}/
|
||||
|
||||
# Convert docker image to aci and stage it
|
||||
echo "Downloading docker2aci ${DOCKER2ACI_VERSION}"
|
||||
wget "https://github.com/appc/docker2aci/releases/download/${DOCKER2ACI_VERSION}/docker2aci-${DOCKER2ACI_VERSION}.tar.gz" &> /dev/null
|
||||
@ -74,13 +59,9 @@ ACI_IMAGE=$(${DOWNLOAD_DIR}/docker2aci-${DOCKER2ACI_VERSION}/docker2aci ${DOCKER
|
||||
cp ${ACI_IMAGE} ${ACI_DIR}/${MOUNTER_ACI_IMAGE}
|
||||
|
||||
# Upload the contents to gcs
|
||||
echo "Uploading rkt artifacts in ${RKT_DIR} to ${RKT_GCS_DIR}"
|
||||
gsutil cp -R ${RKT_DIR} ${RKT_GCS_DIR}
|
||||
echo "Uploading gci mounter ACI in ${ACI_DIR} to ${MOUNTER_GCS_DIR}"
|
||||
gsutil cp ${ACI_DIR}/${MOUNTER_ACI_IMAGE} ${MOUNTER_GCS_DIR}
|
||||
|
||||
echo "Upload completed"
|
||||
echo "Update rkt, stag1-fly.aci & gci-mounter ACI versions and SHA1 in cluster/gce/gci/configure.sh"
|
||||
echo "${RKT_VERSION}/rkt sha1: $(sha1sum ${RKT_DIR}/rkt)"
|
||||
echo "${RKT_VERSION}/stage1-fly.aci sha1: $(sha1sum ${RKT_DIR}/stage1-fly.aci)"
|
||||
echo "Updated gci-mounter ACI version and SHA1 in cluster/gce/gci/configure.sh"
|
||||
echo "${MOUNTER_ACI_IMAGE} hash: $(sha1sum ${ACI_DIR}/${MOUNTER_ACI_IMAGE})"
|
||||
|
||||
4
vendor/k8s.io/kubernetes/cluster/gce/gci/node-helper.sh
generated
vendored
4
vendor/k8s.io/kubernetes/cluster/gce/gci/node-helper.sh
generated
vendored
@ -1,4 +1,4 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
@ -20,6 +20,7 @@ source "${KUBE_ROOT}/cluster/gce/gci/helper.sh"
|
||||
function get-node-instance-metadata {
|
||||
local metadata=""
|
||||
metadata+="kube-env=${KUBE_TEMP}/node-kube-env.yaml,"
|
||||
metadata+="kubelet-config=${KUBE_TEMP}/node-kubelet-config.yaml,"
|
||||
metadata+="user-data=${KUBE_ROOT}/cluster/gce/gci/node.yaml,"
|
||||
metadata+="configure-sh=${KUBE_ROOT}/cluster/gce/gci/configure.sh,"
|
||||
metadata+="cluster-location=${KUBE_TEMP}/cluster-location.txt,"
|
||||
@ -27,6 +28,7 @@ function get-node-instance-metadata {
|
||||
metadata+="gci-update-strategy=${KUBE_TEMP}/gci-update.txt,"
|
||||
metadata+="gci-ensure-gke-docker=${KUBE_TEMP}/gci-ensure-gke-docker.txt,"
|
||||
metadata+="gci-docker-version=${KUBE_TEMP}/gci-docker-version.txt,"
|
||||
metadata+="shutdown-script=${KUBE_ROOT}/cluster/gce/gci/shutdown.sh,"
|
||||
metadata+="${NODE_EXTRA_METADATA}"
|
||||
echo "${metadata}"
|
||||
}
|
||||
|
||||
8
vendor/k8s.io/kubernetes/cluster/gce/gci/node.yaml
generated
vendored
8
vendor/k8s.io/kubernetes/cluster/gce/gci/node.yaml
generated
vendored
@ -40,12 +40,12 @@ write_files:
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
|
||||
- path: /etc/systemd/system/kube-docker-monitor.service
|
||||
- path: /etc/systemd/system/kube-container-runtime-monitor.service
|
||||
permissions: 0644
|
||||
owner: root
|
||||
content: |
|
||||
[Unit]
|
||||
Description=Kubernetes health monitoring for docker
|
||||
Description=Kubernetes health monitoring for container runtime
|
||||
After=kube-node-configuration.service
|
||||
|
||||
[Service]
|
||||
@ -54,7 +54,7 @@ write_files:
|
||||
RemainAfterExit=yes
|
||||
RemainAfterExit=yes
|
||||
ExecStartPre=/bin/chmod 544 /home/kubernetes/bin/health-monitor.sh
|
||||
ExecStart=/home/kubernetes/bin/health-monitor.sh docker
|
||||
ExecStart=/home/kubernetes/bin/health-monitor.sh container-runtime
|
||||
|
||||
[Install]
|
||||
WantedBy=kubernetes.target
|
||||
@ -120,7 +120,7 @@ runcmd:
|
||||
- systemctl daemon-reload
|
||||
- systemctl enable kube-node-installation.service
|
||||
- systemctl enable kube-node-configuration.service
|
||||
- systemctl enable kube-docker-monitor.service
|
||||
- systemctl enable kube-container-runtime-monitor.service
|
||||
- systemctl enable kubelet-monitor.service
|
||||
- systemctl enable kube-logrotate.timer
|
||||
- systemctl enable kube-logrotate.service
|
||||
|
||||
23
vendor/k8s.io/kubernetes/cluster/gce/gci/shutdown.sh
generated
vendored
Executable file
23
vendor/k8s.io/kubernetes/cluster/gce/gci/shutdown.sh
generated
vendored
Executable file
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright 2016 The Kubernetes Authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# A script that let's gci preemptible nodes gracefully terminate in the event of a VM shutdown.
|
||||
preemptible=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/scheduling/preemptible" -H "Metadata-Flavor: Google")
|
||||
if [ ${preemptible} == "TRUE" ]; then
|
||||
echo "Shutting down! Sleeping for a minute to let the node gracefully terminate"
|
||||
# https://cloud.google.com/compute/docs/instances/stopping-or-deleting-an-instance#delete_timeout
|
||||
sleep 30
|
||||
fi
|
||||
Reference in New Issue
Block a user