Initial commit

static-pods/master/api-haproxy.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: k8s-api-haproxy
+  labels:
+    component: k8s-api-haproxy
+    tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-node-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+    - name: api-haproxy
+      image: haproxy:2.4.19-alpine
+{{ if .vars.control_plane.reserve_resources }}
+      resources:
+        requests:
+          cpu: 10m
+          memory: 16Mi
+{{ end }}
+      volumeMounts:
+      - name: config
+        mountPath: /usr/local/etc/haproxy/haproxy.cfg
+        readOnly: true
+    volumes:
+    - name: config
+      hostPath:
+        type: File
+        path: /etc/kubernetes/haproxy-api.cfg
+

static-pods/master/apiserver.yaml

@@ -0,0 +1,114 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: k8s-apiserver
+  annotations:
+    novit.io/bootstrap-prio: "400"
+  labels:
+    component: k8s-apiserver
+    tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-cluster-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+        - name: apiserver
+          image: {{ .vars.k8s_registry}}/kube-apiserver:{{ .vars.kubernetes_version }}
+          command:
+          - kube-apiserver
+          - --secure-port={{ .vars.control_plane.api_port }}
+          - --etcd-servers={{ range $i, $host := hosts_by_group "master" }}{{ if gt $i 0 }},{{end}}https://{{$host.ip}}:2379{{end}}
+          - --etcd-cafile=/tls/etcd-client/ca.crt
+          - --etcd-keyfile=/tls/etcd-client/tls.key
+          - --etcd-certfile=/tls/etcd-client/tls.crt
+          - --cert-dir=/var/lib/kubelet/certs
+          - --allow-privileged=true
+          - --service-cluster-ip-range={{.cluster.subnets.services}}
+          - --client-ca-file=/tls/apiserver/ca.crt
+          - --tls-cert-file=/tls/apiserver/tls.crt
+          - --tls-private-key-file=/tls/apiserver/tls.key
+          - --service-account-issuer=local-server
+          - --service-account-key-file=/tls-ca/service-accounts/ca.key
+          - --service-account-signing-key-file=/tls-ca/service-accounts/ca.key
+          - --proxy-client-key-file=/tls/proxy-client/tls.key
+          - --proxy-client-cert-file=/tls/proxy-client/tls.crt
+          - --requestheader-client-ca-file=/tls/proxy-client/ca.crt
+          - --requestheader-allowed-names=proxy-client
+          - --requestheader-extra-headers-prefix=X-Remote-Extra-
+          - --requestheader-group-headers=X-Remote-Group
+          - --requestheader-username-headers=X-Remote-User
+          - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS
+          - --kubelet-client-certificate=/tls/kubelet-client/tls.crt
+          - --kubelet-client-key=/tls/kubelet-client/tls.key
+          - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota
+          - --token-auth-file=/etc/kubernetes/token-auth.csv
+          - --authorization-mode=RBAC,Node
+          - --event-ttl=6h0m0s
+          - --enable-bootstrap-token-auth
+{{ if .vars.control_plane.reserve_resources }}
+          resources:
+            requests:
+              cpu: 400m
+              memory: 1.2Gi
+{{ end }}
+          volumeMounts:
+          - name: etc-certs
+            mountPath: /etc/ssl/certs
+          - name: tls-etcd-client
+            mountPath: /tls/etcd-client
+          - name: tls-apiserver
+            mountPath: /tls/apiserver
+          - name: tls-kubelet-client
+            mountPath: /tls/kubelet-client
+          - name: ca-cluster
+            mountPath: /tls-ca/cluster
+          - name: ca-service-accounts
+            mountPath: /tls-ca/service-accounts
+          - name: tls-proxy-client
+            mountPath: /tls/proxy-client
+          - name: etc-k8s
+            mountPath: /etc/kubernetes
+          - name: certs
+            mountPath: /var/lib/kubelet/certs
+          #livenessProbe:
+          #    httpGet:
+          #        scheme: HTTPS
+          #        host: 127.0.0.1
+          #        port: {{ .vars.control_plane.api_port }}
+          #        path: /healthz
+          #    initialDelaySeconds: 15
+          #    timeoutSeconds: 15
+          #    failureThreshold: 8
+    volumes:
+        - name: etc-certs
+          hostPath:
+              path: /etc/ssl/certs
+        - name: etc-k8s
+          hostPath:
+              path: /etc/kubernetes
+        - name: tls-etcd-client
+          hostPath:
+              path: /etc/tls/etcd-client
+        - name: ca-cluster
+          hostPath:
+              path: /etc/tls-ca/cluster
+        - name: ca-service-accounts
+          hostPath:
+              path: /etc/tls-ca/service-accounts
+        - name: tls-apiserver
+          hostPath:
+              path: /etc/tls/apiserver
+        - name: tls-kubelet-client
+          hostPath:
+              path: /etc/tls/kubelet-client
+        - name: tls-proxy-client
+          hostPath:
+              path: /etc/tls/proxy-client
+        - name: certs
+          hostPath:
+            path: /var/lib/kubelet/certs

static-pods/master/controller-manager.yaml

@@ -0,0 +1,82 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: k8s-controller-manager
+  annotations:
+    novit.io/bootstrap-prio: "500"
+  labels:
+    component: k8s-controller-manager
+    tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-cluster-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+    - name: controller-manager
+      image: {{ .vars.k8s_registry}}/kube-controller-manager:{{ .vars.kubernetes_version }}
+      command:
+      - kube-controller-manager
+      - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt
+      - --cluster-signing-key-file=/tls-ca/cluster/ca.key
+      - --root-ca-file=/tls-ca/cluster/ca.crt
+      - --service-account-private-key-file=/tls-ca/service-accounts/ca.key
+      - --kubeconfig=/run/k8s/kubeconfig
+      - --allocate-node-cidrs
+      - --cluster-cidr={{ .cluster.subnets.pods }}
+      - --node-cidr-mask-size=24
+      - --controllers=*,bootstrapsigner,tokencleaner
+      - --authentication-kubeconfig=/run/k8s/kubeconfig
+      - --authorization-kubeconfig=/run/k8s/kubeconfig
+{{ if .vars.control_plane.reserve_resources }}
+      resources:
+        requests:
+          cpu: 100m
+          memory: 320Mi
+{{ end }}
+      volumeMounts:
+      - name: config
+        mountPath: /run/k8s
+      - name: etc-k8s
+        mountPath: /etc/kubernetes
+      - name: certs
+        mountPath: /etc/ssl/certs
+      - name: ca-cluster
+        mountPath: /tls-ca/cluster
+      - name: ca-service-accounts
+        mountPath: /tls-ca/service-accounts
+      - name: tls-cluster-client
+        mountPath: /etc/tls/cluster-client
+      livenessProbe:
+          httpGet:
+              scheme: HTTPS
+              host: 127.0.0.1
+              port: 10257
+              path: /healthz
+          initialDelaySeconds: 15
+          timeoutSeconds: 15
+          failureThreshold: 8
+    volumes:
+    - name: config
+      hostPath:
+        path: /etc/kubernetes/control-plane
+    - name: etc-k8s
+      hostPath:
+          path: /etc/kubernetes
+    - name: ca-cluster
+      hostPath:
+          path: /etc/tls-ca/cluster
+    - name: ca-service-accounts
+      hostPath:
+          path: /etc/tls-ca/service-accounts
+    - name: certs
+      hostPath:
+        path: /var/lib/kubelet/certs
+    - name: tls-cluster-client
+      hostPath:
+        path: /etc/tls/cluster-client
+

static-pods/master/docker-registries-mirror.yaml

@@ -0,0 +1,41 @@
+{{- if .vars.docker_registries_mirror_cache.enable }}
+{{- $cache_port := .vars.docker_registries_mirror_cache.port }}
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: docker-registries-mirror
+  annotations:
+    novit.io/bootstrap-prio: "100"
+  labels:
+    app: docker-registries-mirror
+spec:
+  hostNetwork: true
+  dnsPolicy: Default
+  priorityClassName: system-node-critical
+  containers:
+  - name: docker-registries-mirror
+    image: mcluseau/docker-registries-mirror
+    command:
+    - ash
+    - -c
+    - |
+      ALL_HOSTS="{{ if .vars.docker_registries_mirror_cache.remote }}{{ .vars.docker_registries_mirror_cache.remote}}{{ end }}{{ range $i, $host := hosts_by_group "master" }} http://{{$host.ip}}:{{ $cache_port }}{{end}}" \
+      CURRENT_HOST="http://{{ .host.ip }}:{{ $cache_port }}" \
+      OTHER_HOSTS="$(echo ${ALL_HOSTS/${CURRENT_HOST}/})" \
+      sh -c '/bin/docker-registries-mirror -addr=:{{ $cache_port }} -cache-mib={{ .vars.docker_registries_mirror_cache.mb }} -peers=${OTHER_HOSTS/ /,}'
+    {{- if .vars.proxy }}
+    env:
+    - name: HTTP_PROXY
+      value: {{ .vars.proxy }}
+    - name: HTTPS_PROXY
+      value: {{ .vars.proxy }}
+    {{ end }}
+    volumeMounts:
+    - name: cache
+      mountPath: "/cache"
+  volumes:
+  - name: cache
+    hostPath:
+      path: /var/lib/containerd/cache
+{{ end }}

static-pods/master/etcd.yaml

@@ -0,0 +1,86 @@
+apiVersion: v1
+kind: Pod
+metadata:
+    namespace: kube-system
+    name: k8s-etcd
+    annotations:
+      novit.io/bootstrap-prio: "300"
+    labels:
+        component: k8s-etcd
+        tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-cluster-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+        - name: etcd
+          image: {{.vars.etcd.image}}:{{.vars.etcd.version}}
+          command:
+          - etcd
+          - --name={{ .host.name }}
+          - --data-dir=/var/lib/etcd
+          - --trusted-ca-file=/tls/etcd-server/ca.crt
+          - --key-file=/tls/etcd-server/tls.key
+          - --cert-file=/tls/etcd-server/tls.crt
+          - --client-cert-auth=true
+          - --trusted-ca-file=/tls/etcd-server/ca.crt
+          - --listen-client-urls=https://127.0.0.1:2379,https://{{ .host.ip }}:2379
+          - --advertise-client-urls=https://{{ .host.ip }}:2379
+          - --listen-peer-urls=https://{{ .host.ip }}:2380
+          - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt
+          - --peer-key-file=/tls/etcd-peer/tls.key
+          - --peer-cert-file=/tls/etcd-peer/tls.crt
+          - --peer-client-cert-auth=true
+          - --initial-advertise-peer-urls=https://{{ .host.ip }}:2380
+          env:
+          - name:  ETCD_INITIAL_CLUSTER
+            value: {{ range $i, $host := hosts_by_group "master" }}{{ if gt $i 0 }},{{end}}{{$host.name}}=https://{{$host.ip}}:2380{{end}}
+          - name:  ETCD_INITIAL_CLUSTER_STATE
+            value: {{.etcd.cluster_state}}
+          - name:  ETCD_INITIAL_CLUSTER_TOKEN
+            value: '{{ token "etcd-initial-cluster" }}'
+          - name:  ETCDCTL_ENDPOINTS
+            value: {{ range $i, $host := hosts_by_group "master" }}{{ if gt $i 0 }},{{end}}https://{{$host.ip}}:2379{{end}}
+          - name:  ETCDCTL_CACERT
+            value: /tls/etcd-peer/ca.crt
+          - name:  ETCDCTL_CERT
+            value: /tls/etcd-peer/tls.crt
+          - name:  ETCDCTL_KEY
+            value: /tls/etcd-peer/tls.key
+{{ if .vars.control_plane.reserve_resources }}
+          resources:
+            requests:
+              cpu: 200m
+              memory: 1.2Gi
+{{ end }}
+          volumeMounts:
+          - name: etc-certs
+            mountPath: /etc/ssl/certs
+          - name: tls-etcd-server
+            mountPath: /tls/etcd-server
+          - name: tls-etcd-peer
+            mountPath: /tls/etcd-peer
+          - name: k8s
+            mountPath: /etc/kubernetes
+          - name: data
+            mountPath: /var/lib/etcd
+    volumes:
+        - name: etc-certs
+          hostPath:
+              path: /etc/ssl/certs
+        - name: tls-etcd-server
+          hostPath:
+              path: /etc/tls/etcd-server
+        - name: tls-etcd-peer
+          hostPath:
+              path: /etc/tls/etcd-peer
+        - name: k8s
+          hostPath:
+              path: /etc/kubernetes
+        - name: data
+          hostPath:
+              path: /var/lib/etcd

static-pods/master/keepalived.yaml

@@ -0,0 +1,48 @@
+{{- if .vars.public_vip }}
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: k8s-keepalived
+  annotations:
+    novit.io/bootstrap-prio: "250"
+  labels:
+      component: k8s-keepalived
+      tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-node-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+        - name: keepalived
+          image: {{.vars.keepalived.image}}:{{.vars.keepalived.version}}
+          env:
+          - name: KEEPALIVED_AUTH_PASSWORD
+            value: '{{ token "keepalived-vip" }}'
+{{- range $i, $host := hosts_by_group "master" }}
+          - name: KEEPALIVED_UNICAST_PEER_{{$i}}
+            value: {{ $host.ip }}
+{{- end }}
+          - name: KEEPALIVED_VIRTUAL_IPADDRESS_0
+            value: {{.vars.public_vip}}/{{.vars.netmask}}
+          - name: KEEPALIVED_INTERFACE
+            value: {{ .vars.vip_interface }}
+{{- if .vars.keepalived.router_id }}
+          - name: KEEPALIVED_VIRTUAL_ROUTER_ID
+            value: "{{ .vars.keepalived.router_id }}"
+{{- end }}
+          - name: KEEPALIVED_KUBE_APISERVER_CHECK
+            value: "true"
+          - name: KUBE_APISERVER_ADDRESS
+            value: 127.0.0.1
+          - name: KEEPALIVED_GARP_MASTER_REFRESH
+            value: "{{ .vars.keepalived.garp_master_refresh }}"
+          securityContext:
+            capabilities:
+              add:
+              - NET_ADMIN
+{{ end }}

static-pods/master/scheduler.yaml

@@ -0,0 +1,53 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: kube-system
+  name: k8s-scheduler
+  annotations:
+    novit.io/bootstrap-prio: "500"
+  labels:
+    component: k8s-scheduler
+    tier: control-plane
+spec:
+    hostNetwork: true
+    dnsPolicy: Default
+    priorityClassName: system-cluster-critical
+    automountServiceAccountToken: false
+    tolerations:
+    - key: node.kubernetes.io/not-ready
+      effect: NoSchedule
+    containers:
+    - name: scheduler
+      image: {{ .vars.k8s_registry}}/kube-scheduler:{{ .vars.kubernetes_version }}
+      command:
+      - kube-scheduler
+      - --kubeconfig=/run/k8s/kubeconfig
+      - --authentication-kubeconfig=/run/k8s/kubeconfig
+      - --authorization-kubeconfig=/run/k8s/kubeconfig
+      volumeMounts:
+      - name: config
+        mountPath: /run/k8s
+      - name: tls-cluster-client
+        mountPath: /etc/tls/cluster-client
+{{ if .vars.control_plane.reserve_resources }}
+      resources:
+        requests:
+          cpu: 10m
+          memory: 128Mi
+{{ end }}
+      livenessProbe:
+          httpGet:
+              scheme: HTTPS
+              host: 127.0.0.1
+              port: 10259
+              path: /healthz
+          initialDelaySeconds: 15
+          timeoutSeconds: 15
+          failureThreshold: 8
+    volumes:
+    - name: config
+      hostPath:
+        path: /etc/kubernetes/control-plane
+    - name: tls-cluster-client
+      hostPath:
+        path: /etc/tls/cluster-client