diff --git a/cmd/dkl-dir2config/main_test.go b/cmd/dkl-dir2config/main_test.go new file mode 100644 index 0000000..19853bb --- /dev/null +++ b/cmd/dkl-dir2config/main_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +var ( + testDir = "testdata" + goldenFile = "config.yaml.golden" + outFile = "config.yaml" + + binName = "dkl-dir2config" +) + +/* +Build and run the code with default parameters and testdata +*/ +func TestMain(m *testing.M) { + fmt.Println("Building...") + if runtime.GOOS == "windows" { + binName += ".exe" + } + + build := exec.Command("go", "build", "-o", filepath.Join(testDir, binName)) + if err := build.Run(); err != nil { + fmt.Fprintf(os.Stderr, "Cannot build : %v", err) + os.Exit(1) + } + + fmt.Println("Running Tests...") + result := m.Run() + + fmt.Println("Cleaning Up ... ") + os.Remove(binName) + os.Exit(result) +} + +func TestRunMain(t *testing.T) { + err := os.Chdir(testDir) + if err != nil { + t.Fatal(err) + } + + t.Run("RunWithNoArgument", func(t *testing.T) { + cmd := exec.Command("./" + binName) + + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + }) + t.Run("CompareOutputs", func(t *testing.T) { + cmd := exec.Command("./"+binName, "-out", outFile) + + if err := cmd.Run(); err != nil { + t.Fatal(err) + } + + output, err := os.ReadFile(outFile) + if err != nil { + t.Fatal(err) + } + expected, err := os.ReadFile(goldenFile) + if err != nil { + t.Fatal(err) + } + if ret := bytes.Compare(output, expected); ret != 0 { + t.Fatalf("Output (%v) of length %d is different than expected (%v) of length %d", outFile, len(output), goldenFile, len(expected)) + } + }) +} diff --git a/cmd/dkl-dir2config/testdata/cert-requests.yaml b/cmd/dkl-dir2config/testdata/cert-requests.yaml new file mode 100644 index 0000000..420beb5 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/cert-requests.yaml @@ -0,0 +1,39 @@ +- name: etcd-server + ca: etcd + profile: server + per_host: true + template: | + {"CN":"{{.host.name}}","hosts":["127.0.0.1","{{.host.ip}}"],"key":{"algo":"ecdsa","size":256}} +- name: etcd-peer + ca: etcd + profile: peer + per_host: true + template: | + {"CN":"{{.host.name}}","hosts":["127.0.0.1","{{.host.ip}}"],"key":{"algo":"ecdsa","size":256}} +- name: etcd-client + ca: etcd + profile: client + template: | + {"CN":"client","hosts":["*"],"key":{"algo":"ecdsa","size":256}} + +- name: apiserver + ca: cluster + profile: server + per_host: true + template: | + {"CN":"{{.host.name}}","hosts":[ + "kubernetes", "kubernetes.default", "kubernetes.default.svc.{{.cluster.domain}}","{{.host.name}}", + "127.0.0.1","{{.cluster.kubernetes_svc_ip}}","{{.vars.public_vip}}", + {{- if .vars.apiserver_vip }}"{{.vars.apiserver_vip}}",{{ end }} + "{{.host.ip}}" + ],"key":{"algo":"ecdsa","size":521}} +- name: cluster-client + ca: cluster + profile: client + template: | + {"CN":"client","hosts":["*"],"key":{"algo":"ecdsa","size":256}} +- name: kubelet-client + ca: cluster + profile: client + template: | + {"CN":"kubelet-client","names":[{"O":"system:masters"}],"hosts":["*"],"key":{"algo":"ecdsa","size":256}} diff --git a/cmd/dkl-dir2config/testdata/clusters/test.yaml b/cmd/dkl-dir2config/testdata/clusters/test.yaml new file mode 100644 index 0000000..c07e0ca --- /dev/null +++ b/cmd/dkl-dir2config/testdata/clusters/test.yaml @@ -0,0 +1 @@ +from: v1.19:test diff --git a/cmd/dkl-dir2config/testdata/config.yaml b/cmd/dkl-dir2config/testdata/config.yaml new file mode 100644 index 0000000..ccb66f3 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/config.yaml @@ -0,0 +1,6187 @@ +clusters: +- name: test + addons: |+ + --- + # addons: calico + + --- + # Source: calico/templates/calico-config.yaml + # This ConfigMap is used to configure a self-hosted Calico installation. + kind: ConfigMap + apiVersion: v1 + metadata: + name: calico-config + namespace: kube-system + data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } + + --- + # Source: calico/templates/kdd-crds.yaml + + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: bgpconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: bgppeers.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: blockaffinities.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: clusterinformations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: felixconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing intepreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + type: string + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the source address to use on programmed device + routes. By default the source address is left blank, leaving the + kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is legacy. + type: string + iptablesFilterAllowAction: + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeTableRange: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRange specifies the indices of the route tables + that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + type: boolean + vxlanEnabled: + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled. + [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the Wireguard interface. [Default: wg.calico]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by Wireguard. [Default: 51820]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the Wireguard interface. + See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: globalnetworkpolicies.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: globalnetworksets.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: hostendpoints.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamblocks.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + type: string + allocations: + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + type: string + deleted: + type: boolean + strictAffinity: + type: boolean + unallocated: + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamconfigs.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamhandles.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ippools.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 112 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When nat-outgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: networkpolicies.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: networksets.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + --- + # Source: calico/templates/calico-kube-controllers-rbac.yaml + + # Include a clusterrole for the kube-controllers component, + # and bind it to the calico-kube-controllers serviceaccount. + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-kube-controllers + rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-kube-controllers + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers + subjects: + - kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system + --- + + --- + # Source: calico/templates/calico-node-rbac.yaml + # Include a clusterrole for the calico-node DaemonSet, + # and bind it to the calico-node serviceaccount. + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-node + rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: calico-node + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node + subjects: + - kind: ServiceAccount + name: calico-node + namespace: kube-system + + --- + # Source: calico/templates/calico-node.yaml + # This manifest installs the calico-node container, as well + # as the CNI plugins and network config on + # each master and worker node in a Kubernetes cluster. + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node + spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.19.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.19.0 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: docker.io/calico/pod2daemon-flexvol:v3.19.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.19.0 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: IP_AUTODETECTION_METHOD + value: "can-reach=10.105.0.1" + - name: CALICO_IPV4POOL_IPIP + value: "Never" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "192.168.64.0/20" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + --- + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: calico-node + namespace: kube-system + + --- + # Source: calico/templates/calico-kube-controllers.yaml + # See https://github.com/projectcalico/kube-controllers + apiVersion: apps/v1 + kind: Deployment + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.19.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + + --- + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: calico-kube-controllers + namespace: kube-system + + --- + + # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + + apiVersion: policy/v1beta1 + kind: PodDisruptionBudget + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + + --- + # Source: calico/templates/calico-etcd-secrets.yaml + + --- + # Source: calico/templates/calico-typha.yaml + + --- + # Source: calico/templates/configure-canal.yaml + + + + --- + # addons: coredns + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: | + .:53 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" + spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: coredns + template: + metadata: + labels: + k8s-app: coredns + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - store + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.8.3 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + --- + apiVersion: v1 + kind: Service + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" + spec: + selector: + k8s-app: coredns + clusterIP: 192.168.48.2 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + + --- + # addons: ingress-controller + + + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kingress + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: kingress + rules: + - apiGroups: ["","networking.k8s.io"] + resources: + - namespaces + - services + - endpoints + - secrets + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + + + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: kingress + labels: + addonmanager.kubernetes.io/mode: Recreate + subjects: + - kind: ServiceAccount + name: kingress + namespace: kube-system + roleRef: + kind: ClusterRole + name: kingress + apiGroup: rbac.authorization.k8s.io + + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + namespace: kube-system + name: kingress + labels: + k8s-app: kingress + spec: + minReadySeconds: 60 + selector: + matchLabels: + k8s-app: kingress + template: + metadata: + labels: + k8s-app: kingress + spec: + containers: + - args: + - -tls-secret=kube-system/kingress-default-tls + image: mcluseau/kingress:v1.4.1 + imagePullPolicy: IfNotPresent + name: kingress + hostNetwork: true + serviceAccountName: kingress + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: + type: RollingUpdate + + + --- + # addons: knet-wg + + + --- + # addons: kube-proxy + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: kube-system + name: kube-proxy + data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://10.105.32.30:6443 + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-proxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: system:kube-proxy + labels: + addonmanager.kubernetes.io/mode: Reconcile + subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-system + roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io + + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + namespace: kube-system + name: kube-proxy + labels: + k8s-app: kube-proxy + spec: + minReadySeconds: 60 + selector: + matchLabels: + k8s-app: kube-proxy + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + containers: + - command: + - kube-proxy + - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf + - --hostname-override=$(HOSTNAME_OVERRIDE) + - --cluster-cidr=192.168.48.0/20 + - --proxy-mode=iptables + env: + - name: HOSTNAME_OVERRIDE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: k8s.gcr.io/kube-proxy:v1.19.15 + imagePullPolicy: IfNotPresent + name: kube-proxy + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + - mountPath: /lib/modules + name: modules + hostNetwork: true + serviceAccountName: kube-proxy + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: kube-proxy + configMap: + name: kube-proxy + - name: modules + hostPath: + path: /lib/modules + updateStrategy: + type: RollingUpdate + + --- + # addons: kubelet-rubber-stamp + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kubelet-rubber-stamp + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + name: kubelet-rubber-stamp + template: + metadata: + labels: + name: kubelet-rubber-stamp + spec: + serviceAccountName: kubelet-rubber-stamp + tolerations: + - effect: NoSchedule + operator: Exists + priorityClassName: system-cluster-critical + containers: + - name: kubelet-rubber-stamp + image: guillaumefenollar/kubelet-rubber-stamp:v1.0 + args: + - "-v=2" + imagePullPolicy: Always + env: + - name: WATCH_NAMESPACE + value: "" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "kubelet-rubber-stamp" + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubelet-rubber-stamp + subjects: + - kind: ServiceAccount + namespace: kube-system + name: kubelet-rubber-stamp + roleRef: + kind: ClusterRole + name: kubelet-rubber-stamp + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRole + metadata: + name: kubelet-rubber-stamp + rules: + - apiGroups: + - certificates.k8s.io + resources: + - signers + # legacy-unknown: support before kubernetes-1.18.0 + resourceNames: + - "kubernetes.io/legacy-unknown" + - "kubernetes.io/kubelet-serving" + verbs: + - approve + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - create + - update + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kubelet-rubber-stamp + namespace: kube-system + --- + + --- + # addons: kubernetes-dashboard + # Copyright 2017 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + + --- + + kind: Service + apiVersion: v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kube-system + type: Opaque + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kube-system + type: Opaque + data: + csrf: "" + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kube-system + type: Opaque + + --- + + kind: ConfigMap + apiVersion: v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kube-system + + --- + + kind: Role + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + + --- + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + + kind: Deployment + apiVersion: apps/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + image: kubernetesui/dashboard:v2.2.0 + imagePullPolicy: Always + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + - --namespace=kube-system + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + nodeSelector: + "beta.kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + --- + + kind: Service + apiVersion: v1 + metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kube-system + spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper + + --- + + kind: Deployment + apiVersion: apps/v1 + metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kube-system + spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + labels: + k8s-app: dashboard-metrics-scraper + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'runtime/default' + spec: + containers: + - name: dashboard-metrics-scraper + image: kubernetesui/metrics-scraper:v1.0.6 + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + serviceAccountName: kubernetes-dashboard + nodeSelector: + "beta.kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: tmp-volume + emptyDir: {} + --- + + --- + # addons: namespace + apiVersion: v1 + kind: Namespace + metadata: + name: novit-system + spec: + finalizers: + - kubernetes + + --- + # addons: storage-class + + + --- + # addons: tls-bootstrap + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: novit:kubelet-bootstrap + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: auto-approve-renewals-for-nodes + subjects: + - kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + apiGroup: rbac.authorization.k8s.io + + bootstrappods: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-apiserver + tier: control-plane + name: k8s-apiserver + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-apiserver + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-apiserver + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-apiserver + - --etcd-servers= + - --etcd-cafile=/tls/etcd-client/ca.crt + - --etcd-keyfile=/tls/etcd-client/tls.key + - --etcd-certfile=/tls/etcd-client/tls.crt + - --cert-dir=/var/lib/kubelet/certs + - --allow-privileged=true + - --service-cluster-ip-range=192.168.48.0/20 + - --client-ca-file=/tls/apiserver/ca.crt + - --tls-cert-file=/tls/apiserver/tls.crt + - --tls-private-key-file=/tls/apiserver/tls.key + - --service-account-issuer=local-server + - --service-account-key-file=/tls-ca/service-accounts/ca.key + - --service-account-signing-key-file=/tls-ca/service-accounts/ca.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS + - --kubelet-client-certificate=/tls/kubelet-client/tls.crt + - --kubelet-client-key=/tls/kubelet-client/tls.key + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota + - --token-auth-file=/etc/kubernetes/token-auth.csv + - --authorization-mode=RBAC,Node + - --event-ttl=6h0m0s + - --enable-bootstrap-token-auth + image: k8s.gcr.io/kube-apiserver:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: apiserver + resources: + requests: + cpu: 400m + memory: 1.2Gi + volumeMounts: + - mountPath: /etc/ssl/certs + name: etc-certs + - mountPath: /tls/etcd-client + name: tls-etcd-client + - mountPath: /tls/apiserver + name: tls-apiserver + - mountPath: /tls/kubelet-client + name: tls-kubelet-client + - mountPath: /tls-ca/cluster + name: ca-cluster + - mountPath: /tls-ca/service-accounts + name: ca-service-accounts + - mountPath: /etc/kubernetes + name: etc-k8s + - mountPath: /var/lib/kubelet/certs + name: certs + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/ssl/certs + name: etc-certs + - hostPath: + path: /etc/kubernetes + name: etc-k8s + - hostPath: + path: /etc/tls/etcd-client + name: tls-etcd-client + - hostPath: + path: /etc/tls-ca/cluster + name: ca-cluster + - hostPath: + path: /etc/tls-ca/service-accounts + name: ca-service-accounts + - hostPath: + path: /etc/tls/apiserver + name: tls-apiserver + - hostPath: + path: /etc/tls/kubelet-client + name: tls-kubelet-client + - hostPath: + path: /var/lib/kubelet/certs + name: certs + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-controller-manager + tier: control-plane + name: k8s-controller-manager + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-controller-manager + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-controller-manager + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-controller-manager + - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt + - --cluster-signing-key-file=/tls-ca/cluster/ca.key + - --root-ca-file=/tls-ca/cluster/ca.crt + - --service-account-private-key-file=/tls-ca/service-accounts/ca.key + - --kubeconfig=/run/k8s/kubeconfig + - --allocate-node-cidrs + - --cluster-cidr=192.168.64.0/20 + - --node-cidr-mask-size=24 + - --controllers=*,bootstrapsigner,tokencleaner + image: k8s.gcr.io/kube-controller-manager:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: controller-manager + resources: + requests: + cpu: 100m + memory: 320Mi + volumeMounts: + - mountPath: /run/k8s + name: config + - mountPath: /etc/kubernetes + name: etc-k8s + - mountPath: /etc/ssl/certs + name: certs + - mountPath: /tls-ca/cluster + name: ca-cluster + - mountPath: /tls-ca/service-accounts + name: ca-service-accounts + - mountPath: /etc/tls/cluster-client + name: tls-cluster-client + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/kubernetes/control-plane + name: config + - hostPath: + path: /etc/kubernetes + name: etc-k8s + - hostPath: + path: /etc/tls-ca/cluster + name: ca-cluster + - hostPath: + path: /etc/tls-ca/service-accounts + name: ca-service-accounts + - hostPath: + path: /var/lib/kubelet/certs + name: certs + - hostPath: + path: /etc/tls/cluster-client + name: tls-cluster-client + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app: docker-registries-mirror + name: docker-registries-mirror + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + app: docker-registries-mirror + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + app: docker-registries-mirror + spec: + containers: + - command: + - ash + - -c + - | + ALL_HOSTS="" \ + CURRENT_HOST="http://${HOST_IP}:8585" \ + OTHER_HOSTS="$(echo ${ALL_HOSTS/${CURRENT_HOST}/})" \ + sh -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ /,}' + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: mcluseau/docker-registries-mirror + name: docker-registries-mirror + volumeMounts: + - mountPath: /cache + name: cache + hostNetwork: true + priorityClassName: system-node-critical + volumes: + - hostPath: + path: /var/lib/containerd/cache + name: cache + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-etcd + tier: control-plane + name: k8s-etcd + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-etcd + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-etcd + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - etcd + - --name=$(HOSTNAME) + - --data-dir=/var/lib/etcd + - --trusted-ca-file=/tls/etcd-server/ca.crt + - --key-file=/tls/etcd-server/tls.key + - --cert-file=/tls/etcd-server/tls.crt + - --client-cert-auth=true + - --trusted-ca-file=/tls/etcd-server/ca.crt + - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379 + - --advertise-client-urls=https://$(POD_IP):2379 + - --listen-peer-urls=https://$(POD_IP):2380 + - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt + - --peer-key-file=/tls/etcd-peer/tls.key + - --peer-cert-file=/tls/etcd-peer/tls.crt + - --peer-client-cert-auth=true + - --initial-advertise-peer-urls=https://$(POD_IP):2380 + - --initial-cluster-state=existing + - --initial-cluster= + - --initial-cluster-token={{ token "test" "etcd-initial-cluster" }} + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + image: quay.io/coreos/etcd:v3.4.16 + name: etcd + resources: + requests: + cpu: 200m + memory: 1.2Gi + volumeMounts: + - mountPath: /etc/ssl/certs + name: etc-certs + - mountPath: /tls/etcd-server + name: tls-etcd-server + - mountPath: /tls/etcd-peer + name: tls-etcd-peer + - mountPath: /etc/kubernetes + name: k8s + - mountPath: /var/lib/etcd + name: data + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/ssl/certs + name: etc-certs + - hostPath: + path: /etc/tls/etcd-server + name: tls-etcd-server + - hostPath: + path: /etc/tls/etcd-peer + name: tls-etcd-peer + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /var/lib/etcd + name: data + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-keepalived + tier: control-plane + name: k8s-keepalived + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-keepalived + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-keepalived + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KEEPALIVED_AUTH_PASSWORD + value: '{{ token "test" "keepalived-vip" }}' + - name: KEEPALIVED_VIRTUAL_IPADDRESS_0 + value: 10.105.32.30/16 + - name: KEEPALIVED_INTERFACE + value: main + - name: KEEPALIVED_VIRTUAL_ROUTER_ID + value: "1" + - name: KEEPALIVED_KUBE_APISERVER_CHECK + value: "true" + - name: KUBE_APISERVER_ADDRESS + value: 127.0.0.1 + - name: KEEPALIVED_GARP_MASTER_REFRESH + value: "60" + image: novitnc/keepalived:2.0.19 + name: keepalived + securityContext: + capabilities: + add: + - NET_ADMIN + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-scheduler + tier: control-plane + name: k8s-scheduler + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-scheduler + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-scheduler + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-scheduler + - --kubeconfig=/run/k8s/kubeconfig + image: k8s.gcr.io/kube-scheduler:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: scheduler + resources: + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - mountPath: /run/k8s + name: config + - mountPath: /etc/tls/cluster-client + name: tls-cluster-client + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/kubernetes/control-plane + name: config + - hostPath: + path: /etc/tls/cluster-client + name: tls-cluster-client +hosts: +- name: test1 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.1 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.1/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.1/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 4446728c102150aa460dda6a7990c3e3ee345454\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test1\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test1\n ::1 localhost test1\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test1\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test1\" \"server\" \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.1\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test1\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.1\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test1\" \"server\" \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test1\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.1\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.1\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test1:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +- name: test2 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.2 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.2/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.2/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 9e81f0e415796b022c954d36bd9d71953f6bd528\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test2\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test2\n ::1 localhost test2\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test2\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test2\" \"server\" \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.2\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test2\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.2\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test2\" \"server\" \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test2\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.2\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.2\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test2:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +- name: test3 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.3 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.3/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.3/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 4fc7132cd318da9966a61b390a4eb6fa6dcd595a\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test3\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test3\n ::1 localhost test3\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test3\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test3\" \"server\" \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.3\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test3\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.3\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test3\" \"server\" \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test3\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.3\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.3\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test3:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +sslconfig: | + { + "signing": { + "default": { + "expiry": "43800h" + }, + "profiles": { + "server": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth" + ] + }, + "client": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "client auth" + ] + }, + "peer": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } + } diff --git a/cmd/dkl-dir2config/testdata/config.yaml.golden b/cmd/dkl-dir2config/testdata/config.yaml.golden new file mode 100644 index 0000000..ccb66f3 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/config.yaml.golden @@ -0,0 +1,6187 @@ +clusters: +- name: test + addons: |+ + --- + # addons: calico + + --- + # Source: calico/templates/calico-config.yaml + # This ConfigMap is used to configure a self-hosted Calico installation. + kind: ConfigMap + apiVersion: v1 + metadata: + name: calico-config + namespace: kube-system + data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } + + --- + # Source: calico/templates/kdd-crds.yaml + + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: bgpconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: bgppeers.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: blockaffinities.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: clusterinformations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: felixconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enabled" or "Disabled". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing intepreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + type: string + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the source address to use on programmed device + routes. By default the source address is left blank, leaving the + kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is legacy. + type: string + iptablesFilterAllowAction: + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeTableRange: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRange specifies the indices of the route tables + that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + type: boolean + vxlanEnabled: + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled. + [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the Wireguard interface. [Default: wg.calico]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by Wireguard. [Default: 51820]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the Wireguard interface. + See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: globalnetworkpolicies.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: globalnetworksets.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: hostendpoints.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamblocks.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + type: string + allocations: + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + type: string + deleted: + type: boolean + strictAffinity: + type: boolean + unallocated: + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamconfigs.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ipamhandles.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: ippools.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 112 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When nat-outgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: networkpolicies.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and Selector are defined on the same rule, then only workload + endpoints that are matched by both selectors will be selected + by the rule. \n For NetworkPolicy, an empty NamespaceSelector + implies that the Selector is limited to selecting only + workload endpoints in the same namespace as the NetworkPolicy. + \n For NetworkPolicy, `global()` NamespaceSelector implies + that the Selector is limited to selecting only GlobalNetworkSet + or HostEndpoint. \n For GlobalNetworkPolicy, an empty + NamespaceSelector implies the Selector applies to workload + endpoints across all namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: networksets.crd.projectcalico.org + spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true + status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] + + --- + --- + # Source: calico/templates/calico-kube-controllers-rbac.yaml + + # Include a clusterrole for the kube-controllers component, + # and bind it to the calico-kube-controllers serviceaccount. + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-kube-controllers + rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are queried to check for existence. + - apiGroups: [""] + resources: + - pods + verbs: + - get + # IPAM resources are manipulated when nodes are deleted. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - create + - update + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-kube-controllers + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers + subjects: + - kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system + --- + + --- + # Source: calico/templates/calico-node-rbac.yaml + # Include a clusterrole for the calico-node DaemonSet, + # and bind it to the calico-node serviceaccount. + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: calico-node + rules: + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get + + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: calico-node + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node + subjects: + - kind: ServiceAccount + name: calico-node + namespace: kube-system + + --- + # Source: calico/templates/calico-node.yaml + # This manifest installs the calico-node container, as well + # as the CNI plugins and network config on + # each master and worker node in a Kubernetes cluster. + kind: DaemonSet + apiVersion: apps/v1 + metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node + spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.19.0 + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.19.0 + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes + # to communicate with Felix over the Policy Sync API. + - name: flexvol-driver + image: docker.io/calico/pod2daemon-flexvol:v3.19.0 + volumeMounts: + - name: flexvol-driver-host + mountPath: /host/driver + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.19.0 + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: IP_AUTODETECTION_METHOD + value: "can-reach=10.105.0.1" + - name: CALICO_IPV4POOL_IPIP + value: "Never" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "192.168.64.0/20" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + volumeMounts: + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: sysfs + mountPath: /sys/fs/ + # Bidirectional means that, if we mount the BPF filesystem at /sys/fs/bpf it will propagate to the host. + # If the host is known to mount that filesystem already then Bidirectional can be omitted. + mountPropagation: Bidirectional + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sysfs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent + # Used to install Flex Volume Driver + - name: flexvol-driver-host + hostPath: + type: DirectoryOrCreate + path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + --- + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: calico-node + namespace: kube-system + + --- + # Source: calico/templates/calico-kube-controllers.yaml + # See https://github.com/projectcalico/kube-controllers + apiVersion: apps/v1 + kind: Deployment + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.19.0 + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 + + --- + + apiVersion: v1 + kind: ServiceAccount + metadata: + name: calico-kube-controllers + namespace: kube-system + + --- + + # This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + + apiVersion: policy/v1beta1 + kind: PodDisruptionBudget + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + + --- + # Source: calico/templates/calico-etcd-secrets.yaml + + --- + # Source: calico/templates/calico-typha.yaml + + --- + # Source: calico/templates/configure-canal.yaml + + + + --- + # addons: coredns + apiVersion: v1 + kind: ServiceAccount + metadata: + name: coredns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: Reconcile + name: system:coredns + rules: + - apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + addonmanager.kubernetes.io/mode: EnsureExists + name: system:coredns + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns + subjects: + - kind: ServiceAccount + name: coredns + namespace: kube-system + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: coredns + namespace: kube-system + data: + Corefile: | + .:53 { + errors + health + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" + spec: + replicas: 2 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: coredns + template: + metadata: + labels: + k8s-app: coredns + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - store + topologyKey: "kubernetes.io/hostname" + priorityClassName: system-cluster-critical + serviceAccountName: coredns + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: "CriticalAddonsOnly" + operator: "Exists" + containers: + - name: coredns + image: coredns/coredns:1.8.3 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: Corefile + path: Corefile + --- + apiVersion: v1 + kind: Service + metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: coredns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "CoreDNS" + spec: + selector: + k8s-app: coredns + clusterIP: 192.168.48.2 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + + --- + # addons: ingress-controller + + + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kingress + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + + --- + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: kingress + rules: + - apiGroups: ["","networking.k8s.io"] + resources: + - namespaces + - services + - endpoints + - secrets + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update + + + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: kingress + labels: + addonmanager.kubernetes.io/mode: Recreate + subjects: + - kind: ServiceAccount + name: kingress + namespace: kube-system + roleRef: + kind: ClusterRole + name: kingress + apiGroup: rbac.authorization.k8s.io + + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + namespace: kube-system + name: kingress + labels: + k8s-app: kingress + spec: + minReadySeconds: 60 + selector: + matchLabels: + k8s-app: kingress + template: + metadata: + labels: + k8s-app: kingress + spec: + containers: + - args: + - -tls-secret=kube-system/kingress-default-tls + image: mcluseau/kingress:v1.4.1 + imagePullPolicy: IfNotPresent + name: kingress + hostNetwork: true + serviceAccountName: kingress + nodeSelector: + node-role.kubernetes.io/master: "true" + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: + type: RollingUpdate + + + --- + # addons: knet-wg + + + --- + # addons: kube-proxy + apiVersion: v1 + kind: ConfigMap + metadata: + namespace: kube-system + name: kube-proxy + data: + kubeconfig.conf: | + apiVersion: v1 + kind: Config + clusters: + - cluster: + certificate-authority: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + server: https://10.105.32.30:6443 + name: default + contexts: + - context: + cluster: default + namespace: default + user: default + name: default + current-context: default + users: + - name: default + user: + tokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kube-proxy + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1beta1 + metadata: + name: system:kube-proxy + labels: + addonmanager.kubernetes.io/mode: Reconcile + subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-system + roleRef: + kind: ClusterRole + name: system:node-proxier + apiGroup: rbac.authorization.k8s.io + + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + namespace: kube-system + name: kube-proxy + labels: + k8s-app: kube-proxy + spec: + minReadySeconds: 60 + selector: + matchLabels: + k8s-app: kube-proxy + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + containers: + - command: + - kube-proxy + - --kubeconfig=/var/lib/kube-proxy/kubeconfig.conf + - --hostname-override=$(HOSTNAME_OVERRIDE) + - --cluster-cidr=192.168.48.0/20 + - --proxy-mode=iptables + env: + - name: HOSTNAME_OVERRIDE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: k8s.gcr.io/kube-proxy:v1.19.15 + imagePullPolicy: IfNotPresent + name: kube-proxy + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kube-proxy + name: kube-proxy + - mountPath: /lib/modules + name: modules + hostNetwork: true + serviceAccountName: kube-proxy + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + volumes: + - name: kube-proxy + configMap: + name: kube-proxy + - name: modules + hostPath: + path: /lib/modules + updateStrategy: + type: RollingUpdate + + --- + # addons: kubelet-rubber-stamp + apiVersion: apps/v1 + kind: Deployment + metadata: + name: kubelet-rubber-stamp + namespace: kube-system + spec: + replicas: 1 + selector: + matchLabels: + name: kubelet-rubber-stamp + template: + metadata: + labels: + name: kubelet-rubber-stamp + spec: + serviceAccountName: kubelet-rubber-stamp + tolerations: + - effect: NoSchedule + operator: Exists + priorityClassName: system-cluster-critical + containers: + - name: kubelet-rubber-stamp + image: guillaumefenollar/kubelet-rubber-stamp:v1.0 + args: + - "-v=2" + imagePullPolicy: Always + env: + - name: WATCH_NAMESPACE + value: "" + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_NAME + value: "kubelet-rubber-stamp" + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: kubelet-rubber-stamp + subjects: + - kind: ServiceAccount + namespace: kube-system + name: kubelet-rubber-stamp + roleRef: + kind: ClusterRole + name: kubelet-rubber-stamp + apiGroup: rbac.authorization.k8s.io + --- + apiVersion: rbac.authorization.k8s.io/v1beta1 + kind: ClusterRole + metadata: + name: kubelet-rubber-stamp + rules: + - apiGroups: + - certificates.k8s.io + resources: + - signers + # legacy-unknown: support before kubernetes-1.18.0 + resourceNames: + - "kubernetes.io/legacy-unknown" + - "kubernetes.io/kubelet-serving" + verbs: + - approve + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - get + - list + - watch + - apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests/approval + verbs: + - create + - update + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: kubelet-rubber-stamp + namespace: kube-system + --- + + --- + # addons: kubernetes-dashboard + # Copyright 2017 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + + --- + + kind: Service + apiVersion: v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + spec: + ports: + - port: 443 + targetPort: 8443 + selector: + k8s-app: kubernetes-dashboard + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-certs + namespace: kube-system + type: Opaque + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kube-system + type: Opaque + data: + csrf: "" + + --- + + apiVersion: v1 + kind: Secret + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kube-system + type: Opaque + + --- + + kind: ConfigMap + apiVersion: v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kube-system + + --- + + kind: Role + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + rules: + # Allow Dashboard to get, update and delete Dashboard exclusive secrets. + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"] + verbs: ["get", "update", "delete"] + # Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map. + - apiGroups: [""] + resources: ["configmaps"] + resourceNames: ["kubernetes-dashboard-settings"] + verbs: ["get", "update"] + # Allow Dashboard to get metrics. + - apiGroups: [""] + resources: ["services"] + resourceNames: ["heapster", "dashboard-metrics-scraper"] + verbs: ["proxy"] + - apiGroups: [""] + resources: ["services/proxy"] + resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"] + verbs: ["get"] + + --- + + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + rules: + # Allow Metrics Scraper to get metrics from the Metrics server + - apiGroups: ["metrics.k8s.io"] + resources: ["pods", "nodes"] + verbs: ["get", "list", "watch"] + + --- + + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: kubernetes-dashboard + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard + subjects: + - kind: ServiceAccount + name: kubernetes-dashboard + namespace: kube-system + + --- + + kind: Deployment + apiVersion: apps/v1 + metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kube-system + spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - name: kubernetes-dashboard + image: kubernetesui/dashboard:v2.2.0 + imagePullPolicy: Always + ports: + - containerPort: 8443 + protocol: TCP + args: + - --auto-generate-certificates + - --namespace=kube-system + # Uncomment the following line to manually specify Kubernetes API server Host + # If not specified, Dashboard will attempt to auto discover the API server and connect + # to it. Uncomment only if the default does not work. + # - --apiserver-host=http://my-address:port + volumeMounts: + - name: kubernetes-dashboard-certs + mountPath: /certs + # Create on-disk volume to store exec logs + - mountPath: /tmp + name: tmp-volume + livenessProbe: + httpGet: + scheme: HTTPS + path: / + port: 8443 + initialDelaySeconds: 30 + timeoutSeconds: 30 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + volumes: + - name: kubernetes-dashboard-certs + secret: + secretName: kubernetes-dashboard-certs + - name: tmp-volume + emptyDir: {} + serviceAccountName: kubernetes-dashboard + nodeSelector: + "beta.kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + + --- + + kind: Service + apiVersion: v1 + metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kube-system + spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper + + --- + + kind: Deployment + apiVersion: apps/v1 + metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kube-system + spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + labels: + k8s-app: dashboard-metrics-scraper + annotations: + seccomp.security.alpha.kubernetes.io/pod: 'runtime/default' + spec: + containers: + - name: dashboard-metrics-scraper + image: kubernetesui/metrics-scraper:v1.0.6 + ports: + - containerPort: 8000 + protocol: TCP + livenessProbe: + httpGet: + scheme: HTTP + path: / + port: 8000 + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsUser: 1001 + runAsGroup: 2001 + serviceAccountName: kubernetes-dashboard + nodeSelector: + "beta.kubernetes.io/os": linux + # Comment the following tolerations if Dashboard must not be deployed on master + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + volumes: + - name: tmp-volume + emptyDir: {} + --- + + --- + # addons: namespace + apiVersion: v1 + kind: Namespace + metadata: + name: novit-system + spec: + finalizers: + - kubernetes + + --- + # addons: storage-class + + + --- + # addons: tls-bootstrap + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: novit:kubelet-bootstrap + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:node-bootstrapper + subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:bootstrappers + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: auto-approve-renewals-for-nodes + subjects: + - kind: Group + name: system:nodes + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient + apiGroup: rbac.authorization.k8s.io + + bootstrappods: | + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-apiserver + tier: control-plane + name: k8s-apiserver + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-apiserver + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-apiserver + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-apiserver + - --etcd-servers= + - --etcd-cafile=/tls/etcd-client/ca.crt + - --etcd-keyfile=/tls/etcd-client/tls.key + - --etcd-certfile=/tls/etcd-client/tls.crt + - --cert-dir=/var/lib/kubelet/certs + - --allow-privileged=true + - --service-cluster-ip-range=192.168.48.0/20 + - --client-ca-file=/tls/apiserver/ca.crt + - --tls-cert-file=/tls/apiserver/tls.crt + - --tls-private-key-file=/tls/apiserver/tls.key + - --service-account-issuer=local-server + - --service-account-key-file=/tls-ca/service-accounts/ca.key + - --service-account-signing-key-file=/tls-ca/service-accounts/ca.key + - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS + - --kubelet-client-certificate=/tls/kubelet-client/tls.crt + - --kubelet-client-key=/tls/kubelet-client/tls.key + - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota + - --token-auth-file=/etc/kubernetes/token-auth.csv + - --authorization-mode=RBAC,Node + - --event-ttl=6h0m0s + - --enable-bootstrap-token-auth + image: k8s.gcr.io/kube-apiserver:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 6443 + scheme: HTTPS + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: apiserver + resources: + requests: + cpu: 400m + memory: 1.2Gi + volumeMounts: + - mountPath: /etc/ssl/certs + name: etc-certs + - mountPath: /tls/etcd-client + name: tls-etcd-client + - mountPath: /tls/apiserver + name: tls-apiserver + - mountPath: /tls/kubelet-client + name: tls-kubelet-client + - mountPath: /tls-ca/cluster + name: ca-cluster + - mountPath: /tls-ca/service-accounts + name: ca-service-accounts + - mountPath: /etc/kubernetes + name: etc-k8s + - mountPath: /var/lib/kubelet/certs + name: certs + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/ssl/certs + name: etc-certs + - hostPath: + path: /etc/kubernetes + name: etc-k8s + - hostPath: + path: /etc/tls/etcd-client + name: tls-etcd-client + - hostPath: + path: /etc/tls-ca/cluster + name: ca-cluster + - hostPath: + path: /etc/tls-ca/service-accounts + name: ca-service-accounts + - hostPath: + path: /etc/tls/apiserver + name: tls-apiserver + - hostPath: + path: /etc/tls/kubelet-client + name: tls-kubelet-client + - hostPath: + path: /var/lib/kubelet/certs + name: certs + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-controller-manager + tier: control-plane + name: k8s-controller-manager + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-controller-manager + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-controller-manager + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-controller-manager + - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt + - --cluster-signing-key-file=/tls-ca/cluster/ca.key + - --root-ca-file=/tls-ca/cluster/ca.crt + - --service-account-private-key-file=/tls-ca/service-accounts/ca.key + - --kubeconfig=/run/k8s/kubeconfig + - --allocate-node-cidrs + - --cluster-cidr=192.168.64.0/20 + - --node-cidr-mask-size=24 + - --controllers=*,bootstrapsigner,tokencleaner + image: k8s.gcr.io/kube-controller-manager:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10252 + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: controller-manager + resources: + requests: + cpu: 100m + memory: 320Mi + volumeMounts: + - mountPath: /run/k8s + name: config + - mountPath: /etc/kubernetes + name: etc-k8s + - mountPath: /etc/ssl/certs + name: certs + - mountPath: /tls-ca/cluster + name: ca-cluster + - mountPath: /tls-ca/service-accounts + name: ca-service-accounts + - mountPath: /etc/tls/cluster-client + name: tls-cluster-client + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/kubernetes/control-plane + name: config + - hostPath: + path: /etc/kubernetes + name: etc-k8s + - hostPath: + path: /etc/tls-ca/cluster + name: ca-cluster + - hostPath: + path: /etc/tls-ca/service-accounts + name: ca-service-accounts + - hostPath: + path: /var/lib/kubelet/certs + name: certs + - hostPath: + path: /etc/tls/cluster-client + name: tls-cluster-client + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app: docker-registries-mirror + name: docker-registries-mirror + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + app: docker-registries-mirror + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + app: docker-registries-mirror + spec: + containers: + - command: + - ash + - -c + - | + ALL_HOSTS="" \ + CURRENT_HOST="http://${HOST_IP}:8585" \ + OTHER_HOSTS="$(echo ${ALL_HOSTS/${CURRENT_HOST}/})" \ + sh -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ /,}' + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + image: mcluseau/docker-registries-mirror + name: docker-registries-mirror + volumeMounts: + - mountPath: /cache + name: cache + hostNetwork: true + priorityClassName: system-node-critical + volumes: + - hostPath: + path: /var/lib/containerd/cache + name: cache + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-etcd + tier: control-plane + name: k8s-etcd + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-etcd + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-etcd + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - etcd + - --name=$(HOSTNAME) + - --data-dir=/var/lib/etcd + - --trusted-ca-file=/tls/etcd-server/ca.crt + - --key-file=/tls/etcd-server/tls.key + - --cert-file=/tls/etcd-server/tls.crt + - --client-cert-auth=true + - --trusted-ca-file=/tls/etcd-server/ca.crt + - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379 + - --advertise-client-urls=https://$(POD_IP):2379 + - --listen-peer-urls=https://$(POD_IP):2380 + - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt + - --peer-key-file=/tls/etcd-peer/tls.key + - --peer-cert-file=/tls/etcd-peer/tls.crt + - --peer-client-cert-auth=true + - --initial-advertise-peer-urls=https://$(POD_IP):2380 + - --initial-cluster-state=existing + - --initial-cluster= + - --initial-cluster-token={{ token "test" "etcd-initial-cluster" }} + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + image: quay.io/coreos/etcd:v3.4.16 + name: etcd + resources: + requests: + cpu: 200m + memory: 1.2Gi + volumeMounts: + - mountPath: /etc/ssl/certs + name: etc-certs + - mountPath: /tls/etcd-server + name: tls-etcd-server + - mountPath: /tls/etcd-peer + name: tls-etcd-peer + - mountPath: /etc/kubernetes + name: k8s + - mountPath: /var/lib/etcd + name: data + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/ssl/certs + name: etc-certs + - hostPath: + path: /etc/tls/etcd-server + name: tls-etcd-server + - hostPath: + path: /etc/tls/etcd-peer + name: tls-etcd-peer + - hostPath: + path: /etc/kubernetes + name: k8s + - hostPath: + path: /var/lib/etcd + name: data + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-keepalived + tier: control-plane + name: k8s-keepalived + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-keepalived + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-keepalived + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: KEEPALIVED_AUTH_PASSWORD + value: '{{ token "test" "keepalived-vip" }}' + - name: KEEPALIVED_VIRTUAL_IPADDRESS_0 + value: 10.105.32.30/16 + - name: KEEPALIVED_INTERFACE + value: main + - name: KEEPALIVED_VIRTUAL_ROUTER_ID + value: "1" + - name: KEEPALIVED_KUBE_APISERVER_CHECK + value: "true" + - name: KUBE_APISERVER_ADDRESS + value: 127.0.0.1 + - name: KEEPALIVED_GARP_MASTER_REFRESH + value: "60" + image: novitnc/keepalived:2.0.19 + name: keepalived + securityContext: + capabilities: + add: + - NET_ADMIN + hostNetwork: true + priorityClassName: system-node-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + component: k8s-scheduler + tier: control-plane + name: k8s-scheduler + namespace: kube-system + spec: + minReadySeconds: 60 + selector: + matchLabels: + component: k8s-scheduler + tier: control-plane + template: + metadata: + annotations: + node.kubernetes.io/bootstrap-checkpoint: "true" + labels: + component: k8s-scheduler + tier: control-plane + spec: + automountServiceAccountToken: false + containers: + - command: + - kube-scheduler + - --kubeconfig=/run/k8s/kubeconfig + image: k8s.gcr.io/kube-scheduler:v1.19.15 + livenessProbe: + failureThreshold: 8 + httpGet: + host: 127.0.0.1 + path: /healthz + port: 10251 + initialDelaySeconds: 15 + timeoutSeconds: 15 + name: scheduler + resources: + requests: + cpu: 10m + memory: 128Mi + volumeMounts: + - mountPath: /run/k8s + name: config + - mountPath: /etc/tls/cluster-client + name: tls-cluster-client + hostNetwork: true + priorityClassName: system-cluster-critical + tolerations: + - effect: NoSchedule + key: node.kubernetes.io/not-ready + volumes: + - hostPath: + path: /etc/kubernetes/control-plane + name: config + - hostPath: + path: /etc/tls/cluster-client + name: tls-cluster-client +hosts: +- name: test1 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.1 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.1/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.1/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 4446728c102150aa460dda6a7990c3e3ee345454\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test1\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test1\n ::1 localhost test1\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test1\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test1\" \"server\" \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.1\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test1\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.1\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test1\" \"server\" \"\" \"{\\\"CN\\\":\\\"test1\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test1\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.1\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.1\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test1:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +- name: test2 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.2 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.2/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.2/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 9e81f0e415796b022c954d36bd9d71953f6bd528\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test2\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test2\n ::1 localhost test2\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test2\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test2\" \"server\" \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.2\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test2\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.2\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test2\" \"server\" \"\" \"{\\\"CN\\\":\\\"test2\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test2\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.2\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.2\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test2:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +- name: test3 + clustername: test + labels: + node-role.novit.nc/master: "true" + annotations: {} + macs: [] + ips: + - 172.16.0.3 + ipxe: "" + kernel: 5.10.73 + initrd: 1.0.9 + versions: + init: 1.0.7 + kubernetes: v1.19.15_containerd.1.4.11 + modules: 5.10.73 + system: v21.37.0 + config: "root_user:\n password_hash: \"\"\n authorized_keys:\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA3vmeQqioYTDKlEkLlEowdNu/tlS5R7OVKVP6Fo/cb3V3Va5XV+XloSn/JNv7iWm/uDDtB2OG1RSsD+pveoD8MhXt4nkqpZLmwsHgzp7s0zecvg5/37pJxLvPx8u6NZKOJx5Wdl1wyN6OSyXXEtQ5RVVsNzn5WdhdaMc9r+Uv2iWfnKb0mZfOUIdY1UCFqly/AK3nsZXEhjLfOmDQ46mtnamvPIsp1v1gcimQuxyK5MDoC36p1qhMOUHbUBck+oLcaj2AzhtZRtE99a5VAziG4W6t/I1Ok/tb+FXJ7MnLHbsHd54ZLuPy8hTgD3JUmm3lcVp+SGbMtn6oQxyx0Oz9ew== + mcluseau@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7K+fQYrIwO8FOyibaCqgdv0HaMeR1WAa2kCmwRxWmb42Ih1ctbEoHoN1jPlvFZZl6ueZpGCl6aMt79Wa5S9ZBtS47lUcqBtqPd0tQpUyDMNMZH8KfvdEOIEXAFbmqefK4KXTnzkE32k72Tw9zz+iGVDKkH6xmJzuZXFmyn57Cv5K6HFlUu53anWLaBDqPhCLZNOngHaOUSTcNTOA0geGqw4yNgtJegpxxwwIAJVfARN/mZmtMV0ZTc/gIyu7Zc3yjqLmvcm6LgF6ukcDKuQFKU+Th7dDFNgGG4uJT27KVQsnlDRiSWQXSvkIrJFchnOcudciHrskxWO0lyYmiQ5I9 + gfenollar@novit.nc\"\n - \"ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCa1mKSy7buFVNStUixONRYgJzVuQw4SJ8iaCd7O/SffvlgUnhlozmjEXWLY7cIKM42kp5BYPFiyTmOhTHtZnZ0j8EPEaF3zEjXahpsGp3DpgwHMSoeMjk3VL0nlKCAjlggjAW3WtAApvV3KFkABqJqpQg+C8UzbHLVo1tw2fBlCgqPspZmZuAzW1nugr1ZZQITXmYSk5Llaa39XhIRRfrpm6cU4RInc8tKYHP6l5sDVO4eFO65IkNFF/SPxpcPlVFdS/InB7/cW104WbkvKnydxEAKhsAbj85Ga6Amhs6WxVhEaylu3od6x9qKhz3Vi6mYOHkykeIitz8QZpr7QLq3 + cbaillou@novit.nc\"\n\nlayers: # it's TOP to bottom\n- kubernetes\n- init\n- modules\n- + system\n\n\n\nnetworks:\n- match:\n name: lo\n script: |\n ip address add + 127.0.0.1/8 dev lo\n ip -6 address add ::1/128 dev lo\n ip link set lo up\n\n + \ ip link add name main type bond\n ip addr add 172.16.0.3/16 dev main\n + \ ip link set main up\n ip route add default via 10.105.0.1\n\n- match:\n + \ name: en*\n\n ping:\n source: 172.16.0.3/16\n target: 10.105.0.1\n + \ script: |\n ip link set $IFNAME master main\n ip li set $IFNAME up\n\n\n\nstorage:\n + \ udev_match: DEVNAME=/dev/[vsh]da\n remove_volumes:\n - remove-me\n - remove-me-0\n + \ - remove-me-1\n - remove-me-2\n - remove-me-3\n - remove-me-4\n - remove-me-5\n + \ - remove-me-6\n - remove-me-7\n - remove-me-8\n - remove-me-9\n volumes:\n + \ - name: log\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/log\n + \ - name: etcd\n extents: 10%VG\n fs: ext4\n mount:\n path: /var/lib/etcd\n + \ - name: kubelet\n extents: 5%VG\n fs: ext4\n mount:\n path: /var/lib/kubelet\n + \ - name: containerd\n extents: 50%FREE\n fs: ext4\n mount:\n path: + /var/lib/containerd\n\nfiles:\n- path: /etc/machine-id\n content: |\n 4fc7132cd318da9966a61b390a4eb6fa6dcd595a\n\n- + path: /etc/rc.conf\n content: |\n rc_shell=/sbin/sulogin\n rc_logger=\"YES\"\n + \ #rc_log_path=\"/var/log/rc.log\"\n unicode=\"YES\"\n rc_tty_number=12\n\n + \ rc_cgroup_mode=\"legacy\"\n rc_cgroup_memory_use_hierarchy=\"YES\"\n rc_controller_cgroups=\"YES\"\n\n- + path: /etc/hostname\n content: test3\n\n- path: /etc/hosts\n content: |\n 127.0.0.1 + localhost test3\n ::1 localhost test3\n\n 10.105.32.30 kubernetes\n\n\n- + path: /etc/resolv.conf\n content: |\n nameserver 10.105.0.1\n \n\n- path: + /etc/sysctl.conf\n content: |\n fs.file-max = 20971520\n fs.inotify.max_user_watches + = 1048576\n kernel.pid_max = 1048576\n net.ipv4.ip_forward = 1\n vm.max_map_count + = 262144\n\n net.ipv4.neigh.default.gc_thresh1 = 16384\n net.ipv4.neigh.default.gc_thresh2 + = 28672\n net.ipv4.neigh.default.gc_thresh3 = 32768\n\n \n\n- path: /etc/udev/rules.d/50-io-scheduler.rules\n + \ content: |\n ACTION==\"add|change\", KERNEL==\"sd[a-z]\", ATTR{queue/scheduler}=\"bfq\", + ATTR{queue/nr_requests}=\"1024\"\n ACTION==\"add|change\", KERNEL==\"nvme[0-9]n[0-9]\", + ATTR{queue/scheduler}=\"bfq\", ATTR{queue/nr_requests}=\"2048\"\n\n\n\n# -------------------------------------------------------------------------\n{{ + ssh_host_keys \"/etc/ssh\" \"test\" \"test3\"}}\n\n# ------------------------------------------------------------------------\n\n# + certificates for etcd servers\n{{ tls_dir \"/etc/tls/etcd-server\" \"test\" \"etcd\" + \"etcd-server/test3\" \"server\" \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.3\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n{{ tls_dir \"/etc/tls/etcd-peer\" \"test\" \"etcd\" \"etcd-peer/test3\" \"peer\" + \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\\"127.0.0.1\\\",\\\"172.16.0.3\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# certificates for etcd clients\n{{ tls_dir \"/etc/tls/etcd-client\" \"test\" + \"etcd\" \"etcd-client\" \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n# cluster certificates\n{{ ca_dir \"test\" \"cluster\" }}\n{{ ca_dir \"test\" + \"service-accounts\" }}\n\n{{ tls_dir \"/etc/tls/apiserver\" \"test\" \"cluster\" + \"apiserver/test3\" \"server\" \"\" \"{\\\"CN\\\":\\\"test3\\\",\\\"hosts\\\":[\\n + \ \\\"kubernetes\\\", \\\"kubernetes.default\\\", \\\"kubernetes.default.svc.cluster.local\\\",\\\"test3\\\",\\n + \ \\\"127.0.0.1\\\",\\\"192.168.48.1\\\",\\\"10.105.32.30\\\",\\n \\\"172.16.0.3\\\"\\n],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":521}}\\n\" + }}\n{{ tls_dir \"/etc/tls/kubelet-client\" \"test\" \"cluster\" \"kubelet-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"kubelet-client\\\",\\\"names\\\":[{\\\"O\\\":\\\"system:masters\\\"}],\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n\n\n{{ tls_dir \"/etc/tls/cluster-client\" \"test\" \"cluster\" \"cluster-client\" + \"client\" \"\" \"{\\\"CN\\\":\\\"client\\\",\\\"hosts\\\":[\\\"*\\\"],\\\"key\\\":{\\\"algo\\\":\\\"ecdsa\\\",\\\"size\\\":256}}\\n\" + }}\n\n- path: /etc/kubernetes/token-auth.csv\n mode: 0600\n content: |\n {{ + token \"test\" \"bootstrap\" }},kubelet-bootstrap,10001,\"system:bootstrappers\"\n + \ {{ token \"test\" \"admin\" }},admin-token,10002,\"system:masters\"\n\n# + ------------------------------------------------------------------------\n- path: + /etc/chrony/chrony.conf\n mode: 0644\n content: |\n server 0.gentoo.pool.ntp.org + iburst\n server 1.gentoo.pool.ntp.org iburst\n server 2.gentoo.pool.ntp.org + iburst\n server 3.gentoo.pool.ntp.org iburst\n\n driftfile /var/lib/chrony/drift\n + \ makestep 1.0 3\n rtcsync\n \n\n\n\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/k8s-local-volumes\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n while true\n do\n for + dev in /dev/k8s-pv/* /dev/storage/k8s-pv-*\n do\n [ -e $dev + ] || continue\n\n mp=${dev/dev/mnt}\n mkdir -p $mp\n findmnt + $dev >/dev/null || {\n fsck -p $dev &&\n mount $dev + $mp\n }\n done\n\n sleep 10\n done\n\n # + ------------------------------------------------------------------------\n- path: + /etc/direktil/services/kubelet-network-bug\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n\n path=/var/log/kubelet.log\n while + true;\n do\n closed_connection=$(/usr/bin/tail $path | grep \"use of closed + connection\")\n if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Closed + connection error not found in logs\" >> $path\n elif [[ $closed_connection + ]]; then\n echo \"KUBELET CHECK - Closed connection error detected, restarting + kubelet...\" $path\n /usr/bin/killall kubelet\n fi\n\n empty_error=$(/usr/bin/tail + $path | grep 'an error on the server (\"\") has prevented the request from succeeding')\n + \ if [[ $? != 0 ]]; then\n echo \"KUBELET CHECK - Empty error message + error not found in logs\" >> $path\n elif [[ $empty_error ]]; then\n echo + \"KUBELET CHECK - Empty error message detected, restarting kubelet...\" $path\n + \ /usr/bin/killall kubelet\n fi\n\n sleep 1m\n done\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/containerd\n mode: 0755\n content: |\n #! + /bin/sh\n # ---\n # restart: 3\n # provides:\n # - k8s-runtime\n\n + \ set -ex\n\n ulimit -n 1048576\n ulimit -u unlimited\n ulimit + -c unlimited\n\n \n\n exec /usr/bin/containerd \\\n --log-level + info\n\n# ------------------------------------------------------------------------\n- + path: /etc/direktil/services/kubelet\n mode: 0755\n content: |\n #! /bin/sh\n + \ # ---\n # restart: 3\n # needs:\n # - k8s-runtime\n set + -ex\n\n ctr_sock=\"/run/containerd/containerd.sock\"\n echo \"waiting + for $ctr_sock\"\n while ! [ -e $ctr_sock ]; do sleep 1; done\n\n #ulimit + -n 1048576\n mkdir -p /var/lib/kubelet/manifests\n\n exec /usr/bin/kubelet + \\\n --config=/etc/kubernetes/kubelet.yaml \\\n --dynamic-config-dir=/var/lib/kubelet/dynamic-config + \\\n --node-labels=node-role.novit.nc/master=true \\\n --network-plugin=cni + \\\n --container-runtime=remote \\\n --container-runtime-endpoint=unix://$ctr_sock + \\\n --bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \\\n --kubeconfig=/var/lib/kubelet/kubeconfig + \\\n --node-ip=172.16.0.3\n\n\n\n# -------------------------------------------------------------------------\n\n- + path: /etc/containerd/config.toml\n mode: 0600\n content: |\n version = 2\n + \ root = \"/var/lib/containerd\"\n state = \"/run/containerd\"\n plugin_dir + = \"\"\n disabled_plugins = []\n required_plugins = []\n oom_score = + 0\n\n [grpc]\n address = \"/run/containerd/containerd.sock\"\n tcp_address + = \"\"\n tcp_tls_cert = \"\"\n tcp_tls_key = \"\"\n uid = 0\n gid + = 0\n max_recv_message_size = 16777216\n max_send_message_size = 16777216\n\n + \ [ttrpc]\n address = \"\"\n uid = 0\n gid = 0\n\n [debug]\n + \ address = \"\"\n uid = 0\n gid = 0\n level = \"\"\n\n [metrics]\n + \ address = \"\"\n grpc_histogram = false\n\n [cgroup]\n path + = \"\"\n\n [timeouts]\n \"io.containerd.timeout.shim.cleanup\" = \"5s\"\n + \ \"io.containerd.timeout.shim.load\" = \"5s\"\n \"io.containerd.timeout.shim.shutdown\" + = \"3s\"\n \"io.containerd.timeout.task.state\" = \"2s\"\n\n [plugins]\n + \ [plugins.\"io.containerd.gc.v1.scheduler\"]\n pause_threshold = 0.02\n + \ deletion_threshold = 0\n mutation_threshold = 100\n schedule_delay + = \"0s\"\n startup_delay = \"100ms\"\n [plugins.\"io.containerd.grpc.v1.cri\"]\n + \ disable_tcp_service = true\n stream_server_address = \"127.0.0.1\"\n + \ stream_server_port = \"0\"\n stream_idle_timeout = \"4h0m0s\"\n + \ enable_selinux = false\n sandbox_image = \"k8s.gcr.io/pause:3.1\"\n + \ stats_collect_period = 10\n systemd_cgroup = false\n enable_tls_streaming + = false\n max_container_log_line_size = 16384\n disable_cgroup = + false\n disable_apparmor = false\n restrict_oom_score_adj = false\n + \ max_concurrent_downloads = 3\n disable_proc_mount = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd]\n + \ snapshotter = \"overlayfs\"\n default_runtime_name = \"runc\"\n + \ no_pivot = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.default_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.untrusted_workload_runtime]\n + \ runtime_type = \"\"\n runtime_engine = \"\"\n runtime_root + = \"\"\n privileged_without_host_devices = false\n [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".containerd.runtimes.runc]\n + \ runtime_type = \"io.containerd.runc.v1\"\n runtime_engine + = \"\"\n runtime_root = \"\"\n privileged_without_host_devices + = false\n [plugins.\"io.containerd.grpc.v1.cri\".cni]\n bin_dir + = \"/opt/cni/bin\"\n conf_dir = \"/etc/cni/net.d\"\n max_conf_num + = 1\n conf_template = \"\"\n [plugins.\"io.containerd.grpc.v1.cri\".registry]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"docker.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/registry-1.docker.io/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"k8s.gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/k8s.gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"gcr.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/gcr.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"quay.io\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/quay.io/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.isi.nc/v2\"]\n [plugins.\"io.containerd.grpc.v1.cri\".registry.mirrors.\"dkr.dev.isi.nc\"]\n + \ endpoint = [\"http://127.0.0.1:8585/https/dkr.dev.isi.nc/v2\"]\n + \ [plugins.\"io.containerd.grpc.v1.cri\".x509_key_pair_streaming]\n tls_cert_file + = \"\"\n tls_key_file = \"\"\n [plugins.\"io.containerd.internal.v1.opt\"]\n + \ path = \"/opt/containerd\"\n [plugins.\"io.containerd.internal.v1.restart\"]\n + \ interval = \"10s\"\n [plugins.\"io.containerd.metadata.v1.bolt\"]\n + \ content_sharing_policy = \"shared\"\n [plugins.\"io.containerd.monitor.v1.cgroups\"]\n + \ no_prometheus = false\n [plugins.\"io.containerd.runtime.v1.linux\"]\n + \ shim = \"containerd-shim\"\n runtime = \"runc\"\n runtime_root + = \"\"\n no_shim = false\n shim_debug = false\n [plugins.\"io.containerd.runtime.v2.task\"]\n + \ platforms = [\"linux/amd64\"]\n [plugins.\"io.containerd.service.v1.diff-service\"]\n + \ default = [\"walking\"]\n [plugins.\"io.containerd.snapshotter.v1.devmapper\"]\n + \ root_path = \"\"\n pool_name = \"\"\n base_image_size = + \"\"\n\n\n# -------------------------------------------------------------------------\n- + path: /etc/kubernetes/kubelet.yaml\n mode: 0600\n content: |\n kind: KubeletConfiguration\n + \ apiVersion: kubelet.config.k8s.io/v1beta1\n staticPodPath: /var/lib/kubelet/manifests\n\n + \ clusterDomain: cluster.local\n clusterDNS:\n - 192.168.48.2\n + \ podCIDR: 192.168.64.0/20\n\n address: 0.0.0.0\n authentication:\n + \ x509:\n clientCAFile: /etc/tls/cluster-client/ca.crt\n anonymous:\n + \ enabled: false\n maxPods: 220\n serializeImagePulls: false\n + \ featureGates:\n DynamicKubeletConfig: true\n RotateKubeletClientCertificate: + true\n RotateKubeletServerCertificate: true\n CRIContainerLogRotation: + true\n SupportPodPidsLimit: true\n\n serverTLSBootstrap: true\n rotateCertificates: + true\n\n podPidsLimit: 4096\n\n containerLogMaxFiles: 2\n containerLogMaxSize: + 16Mi\n\n # cgroups configuration\n cgroupsPerQOS: true\n cgroupDriver: + cgroupfs\n\n systemReservedCgroup: openrc\n systemReserved:\n cpu: + \ \"100m\"\n memory: \"256Mi\"\n\n kubeReservedCgroup: podruntime\n + \ kubeReserved:\n cpu: \"200m\"\n memory: \"256Mi\"\n\n #evictionHard:\n + \ # memory.available: 100Mi\n\n- path: /etc/kubernetes/bootstrap.kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://kubernetes:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: kubelet-bootstrap\n + \ name: local\n users:\n - name: kubelet-bootstrap\n user:\n token: + {{ token \"test\" \"bootstrap\" }}\n\n- path: /etc/kubernetes/control-plane/kubeconfig\n + \ mode: 0600\n content: |\n apiVersion: v1\n kind: Config\n preferences: + {}\n current-context: local\n clusters:\n - cluster:\n certificate-authority: + /etc/tls/cluster-client/ca.crt\n server: https://127.0.0.1:6443\n name: + local\n contexts:\n - context:\n cluster: local\n user: control-plane\n + \ name: local\n users:\n - name: control-plane\n user:\n token: + {{ token \"test\" \"admin\" }}\n\n\n\n\n\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-apiserver.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-apiserver\n tier: control-plane\n + \ name: k8s-apiserver\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - kube-apiserver\n - + --etcd-servers=\n - --etcd-cafile=/tls/etcd-client/ca.crt\n - --etcd-keyfile=/tls/etcd-client/tls.key\n + \ - --etcd-certfile=/tls/etcd-client/tls.crt\n - --cert-dir=/var/lib/kubelet/certs\n + \ - --allow-privileged=true\n - --service-cluster-ip-range=192.168.48.0/20\n + \ - --client-ca-file=/tls/apiserver/ca.crt\n - --tls-cert-file=/tls/apiserver/tls.crt\n + \ - --tls-private-key-file=/tls/apiserver/tls.key\n - --service-account-issuer=local-server\n + \ - --service-account-key-file=/tls-ca/service-accounts/ca.key\n - + --service-account-signing-key-file=/tls-ca/service-accounts/ca.key\n - + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS\n + \ - --kubelet-client-certificate=/tls/kubelet-client/tls.crt\n - + --kubelet-client-key=/tls/kubelet-client/tls.key\n - --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota\n + \ - --token-auth-file=/etc/kubernetes/token-auth.csv\n - --authorization-mode=RBAC,Node\n + \ - --event-ttl=6h0m0s\n - --enable-bootstrap-token-auth\n image: + k8s.gcr.io/kube-apiserver:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 6443\n scheme: HTTPS\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: apiserver\n resources:\n + \ requests:\n cpu: 400m\n memory: 1.2Gi\n volumeMounts:\n + \ - mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: + /tls/etcd-client\n name: tls-etcd-client\n - mountPath: /tls/apiserver\n + \ name: tls-apiserver\n - mountPath: /tls/kubelet-client\n name: + tls-kubelet-client\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n + \ - mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n + \ - mountPath: /etc/kubernetes\n name: etc-k8s\n - mountPath: + /var/lib/kubelet/certs\n name: certs\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/kubernetes\n + \ name: etc-k8s\n - hostPath:\n path: /etc/tls/etcd-client\n + \ name: tls-etcd-client\n - hostPath:\n path: /etc/tls-ca/cluster\n + \ name: ca-cluster\n - hostPath:\n path: /etc/tls-ca/service-accounts\n + \ name: ca-service-accounts\n - hostPath:\n path: /etc/tls/apiserver\n + \ name: tls-apiserver\n - hostPath:\n path: /etc/tls/kubelet-client\n + \ name: tls-kubelet-client\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-controller-manager.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-controller-manager\n tier: control-plane\n + \ name: k8s-controller-manager\n namespace: kube-system\n spec:\n + \ automountServiceAccountToken: false\n containers:\n - command:\n + \ - kube-controller-manager\n - --cluster-signing-cert-file=/tls-ca/cluster/ca.crt\n + \ - --cluster-signing-key-file=/tls-ca/cluster/ca.key\n - --root-ca-file=/tls-ca/cluster/ca.crt\n + \ - --service-account-private-key-file=/tls-ca/service-accounts/ca.key\n + \ - --kubeconfig=/run/k8s/kubeconfig\n - --allocate-node-cidrs\n + \ - --cluster-cidr=192.168.64.0/20\n - --node-cidr-mask-size=24\n + \ - --controllers=*,bootstrapsigner,tokencleaner\n image: k8s.gcr.io/kube-controller-manager:v1.19.15\n + \ livenessProbe:\n failureThreshold: 8\n httpGet:\n host: + 127.0.0.1\n path: /healthz\n port: 10252\n initialDelaySeconds: + 15\n timeoutSeconds: 15\n name: controller-manager\n resources:\n + \ requests:\n cpu: 100m\n memory: 320Mi\n volumeMounts:\n + \ - mountPath: /run/k8s\n name: config\n - mountPath: /etc/kubernetes\n + \ name: etc-k8s\n - mountPath: /etc/ssl/certs\n name: + certs\n - mountPath: /tls-ca/cluster\n name: ca-cluster\n - + mountPath: /tls-ca/service-accounts\n name: ca-service-accounts\n - + mountPath: /etc/tls/cluster-client\n name: tls-cluster-client\n hostNetwork: + true\n priorityClassName: system-cluster-critical\n tolerations:\n - + effect: NoSchedule\n key: node.kubernetes.io/not-ready\n volumes:\n + \ - hostPath:\n path: /etc/kubernetes/control-plane\n name: + config\n - hostPath:\n path: /etc/kubernetes\n name: etc-k8s\n + \ - hostPath:\n path: /etc/tls-ca/cluster\n name: ca-cluster\n + \ - hostPath:\n path: /etc/tls-ca/service-accounts\n name: + ca-service-accounts\n - hostPath:\n path: /var/lib/kubelet/certs\n + \ name: certs\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_docker-registries-mirror.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n app: docker-registries-mirror\n name: docker-registries-mirror\n + \ namespace: kube-system\n spec:\n containers:\n - command:\n + \ - ash\n - -c\n - |\n ALL_HOSTS=\"\" \\\n CURRENT_HOST=\"http://${HOST_IP}:8585\" + \\\n OTHER_HOSTS=\"$(echo ${ALL_HOSTS/${CURRENT_HOST}/})\" \\\n sh + -c '/bin/docker-registries-mirror -addr=0.0.0.0:8585 -cache-mib=20000 -peers=${OTHER_HOSTS/ + /,}'\n env:\n - name: HOST_IP\n valueFrom:\n fieldRef:\n + \ fieldPath: status.hostIP\n image: mcluseau/docker-registries-mirror\n + \ name: docker-registries-mirror\n volumeMounts:\n - mountPath: + /cache\n name: cache\n hostNetwork: true\n priorityClassName: + system-node-critical\n volumes:\n - hostPath:\n path: /var/lib/containerd/cache\n + \ name: cache\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-etcd.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-etcd\n tier: control-plane\n name: + k8s-etcd\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - command:\n - etcd\n - --name=$(HOSTNAME)\n + \ - --data-dir=/var/lib/etcd\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --key-file=/tls/etcd-server/tls.key\n - --cert-file=/tls/etcd-server/tls.crt\n + \ - --client-cert-auth=true\n - --trusted-ca-file=/tls/etcd-server/ca.crt\n + \ - --listen-client-urls=https://127.0.0.1:2379,https://$(POD_IP):2379\n + \ - --advertise-client-urls=https://$(POD_IP):2379\n - --listen-peer-urls=https://$(POD_IP):2380\n + \ - --peer-trusted-ca-file=/tls/etcd-peer/ca.crt\n - --peer-key-file=/tls/etcd-peer/tls.key\n + \ - --peer-cert-file=/tls/etcd-peer/tls.crt\n - --peer-client-cert-auth=true\n + \ - --initial-advertise-peer-urls=https://$(POD_IP):2380\n - --initial-cluster-state=existing\n + \ - --initial-cluster=\n - --initial-cluster-token={{ token \"test\" + \"etcd-initial-cluster\" }}\n env:\n - name: HOSTNAME\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + spec.nodeName\n - name: POD_IP\n valueFrom:\n fieldRef:\n + \ apiVersion: v1\n fieldPath: status.podIP\n image: + quay.io/coreos/etcd:v3.4.16\n name: etcd\n resources:\n requests:\n + \ cpu: 200m\n memory: 1.2Gi\n volumeMounts:\n - + mountPath: /etc/ssl/certs\n name: etc-certs\n - mountPath: /tls/etcd-server\n + \ name: tls-etcd-server\n - mountPath: /tls/etcd-peer\n name: + tls-etcd-peer\n - mountPath: /etc/kubernetes\n name: k8s\n - + mountPath: /var/lib/etcd\n name: data\n hostNetwork: true\n priorityClassName: + system-cluster-critical\n tolerations:\n - effect: NoSchedule\n key: + node.kubernetes.io/not-ready\n volumes:\n - hostPath:\n path: + /etc/ssl/certs\n name: etc-certs\n - hostPath:\n path: /etc/tls/etcd-server\n + \ name: tls-etcd-server\n - hostPath:\n path: /etc/tls/etcd-peer\n + \ name: tls-etcd-peer\n - hostPath:\n path: /etc/kubernetes\n + \ name: k8s\n - hostPath:\n path: /var/lib/etcd\n name: + data\n- path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-keepalived.yaml\n + \ mode: 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n + \ labels:\n component: k8s-keepalived\n tier: control-plane\n + \ name: k8s-keepalived\n namespace: kube-system\n spec:\n automountServiceAccountToken: + false\n containers:\n - env:\n - name: POD_IP\n valueFrom:\n + \ fieldRef:\n apiVersion: v1\n fieldPath: + status.podIP\n - name: KEEPALIVED_AUTH_PASSWORD\n value: '{{ token + \"test\" \"keepalived-vip\" }}'\n - name: KEEPALIVED_VIRTUAL_IPADDRESS_0\n + \ value: 10.105.32.30/16\n - name: KEEPALIVED_INTERFACE\n value: + main\n - name: KEEPALIVED_VIRTUAL_ROUTER_ID\n value: \"1\"\n - + name: KEEPALIVED_KUBE_APISERVER_CHECK\n value: \"true\"\n - name: + KUBE_APISERVER_ADDRESS\n value: 127.0.0.1\n - name: KEEPALIVED_GARP_MASTER_REFRESH\n + \ value: \"60\"\n image: novitnc/keepalived:2.0.19\n name: + keepalived\n securityContext:\n capabilities:\n add:\n + \ - NET_ADMIN\n hostNetwork: true\n priorityClassName: system-node-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n- + path: /etc/kubernetes/manifests.bootstrap/kube-system_k8s-scheduler.yaml\n mode: + 416\n content: |\n apiVersion: v1\n kind: Pod\n metadata:\n labels:\n + \ component: k8s-scheduler\n tier: control-plane\n name: k8s-scheduler\n + \ namespace: kube-system\n spec:\n automountServiceAccountToken: false\n + \ containers:\n - command:\n - kube-scheduler\n - --kubeconfig=/run/k8s/kubeconfig\n + \ image: k8s.gcr.io/kube-scheduler:v1.19.15\n livenessProbe:\n failureThreshold: + 8\n httpGet:\n host: 127.0.0.1\n path: /healthz\n + \ port: 10251\n initialDelaySeconds: 15\n timeoutSeconds: + 15\n name: scheduler\n resources:\n requests:\n cpu: + 10m\n memory: 128Mi\n volumeMounts:\n - mountPath: /run/k8s\n + \ name: config\n - mountPath: /etc/tls/cluster-client\n name: + tls-cluster-client\n hostNetwork: true\n priorityClassName: system-cluster-critical\n + \ tolerations:\n - effect: NoSchedule\n key: node.kubernetes.io/not-ready\n + \ volumes:\n - hostPath:\n path: /etc/kubernetes/control-plane\n + \ name: config\n - hostPath:\n path: /etc/tls/cluster-client\n + \ name: tls-cluster-client\n\n\n\n- path: /etc/cron.monthly/clean-archives-logs\n + \ mode: 0755\n content: |\n #! /bin/bash\n find /var/log/archives/ + -type f -mtime +20 -delete\n\n- path: /root/host-checks.sh\n mode: 0700\n content: + |\n #! /bin/bash\n\n echo \"kubelet health:\"\n curl --cacert /etc/tls/kubelet-client/ca.crt + \\\n --cert /etc/tls/kubelet-client/tls.crt \\\n --key /etc/tls/kubelet-client/tls.key + \\\n https://test3:10250/healthz\n echo\n echo \"API health:\"\n + \ curl --cacert /etc/tls/apiserver/ca.crt \\\n --cert /etc/tls/cluster-client/tls.crt + \\\n --key /etc/tls/cluster-client/tls.key \\\n https://127.0.0.1:6443/healthz\n + \ echo\n\n" +sslconfig: | + { + "signing": { + "default": { + "expiry": "43800h" + }, + "profiles": { + "server": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth" + ] + }, + "client": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "client auth" + ] + }, + "peer": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } + } diff --git a/cmd/dkl-dir2config/testdata/defaults b/cmd/dkl-dir2config/testdata/defaults new file mode 160000 index 0000000..be8c859 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/defaults @@ -0,0 +1 @@ +Subproject commit be8c8592fbc39267755978860a095cbf297e924d diff --git a/cmd/dkl-dir2config/testdata/groups/test-master.yaml b/cmd/dkl-dir2config/testdata/groups/test-master.yaml new file mode 100644 index 0000000..98bc56d --- /dev/null +++ b/cmd/dkl-dir2config/testdata/groups/test-master.yaml @@ -0,0 +1 @@ +from: v1.19:master diff --git a/cmd/dkl-dir2config/testdata/hosts/test1.yaml b/cmd/dkl-dir2config/testdata/hosts/test1.yaml new file mode 100644 index 0000000..ca31613 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/hosts/test1.yaml @@ -0,0 +1,3 @@ +ip: 172.16.0.1 +cluster: test +group: test-master diff --git a/cmd/dkl-dir2config/testdata/hosts/test2.yaml b/cmd/dkl-dir2config/testdata/hosts/test2.yaml new file mode 100644 index 0000000..5345946 --- /dev/null +++ b/cmd/dkl-dir2config/testdata/hosts/test2.yaml @@ -0,0 +1,3 @@ +ip: 172.16.0.2 +cluster: test +group: test-master diff --git a/cmd/dkl-dir2config/testdata/hosts/test3.yaml b/cmd/dkl-dir2config/testdata/hosts/test3.yaml new file mode 100644 index 0000000..f72120f --- /dev/null +++ b/cmd/dkl-dir2config/testdata/hosts/test3.yaml @@ -0,0 +1,3 @@ +ip: 172.16.0.3 +cluster: test +group: test-master diff --git a/cmd/dkl-dir2config/testdata/ssl-config.json b/cmd/dkl-dir2config/testdata/ssl-config.json new file mode 100644 index 0000000..dba73de --- /dev/null +++ b/cmd/dkl-dir2config/testdata/ssl-config.json @@ -0,0 +1,34 @@ +{ + "signing": { + "default": { + "expiry": "43800h" + }, + "profiles": { + "server": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth" + ] + }, + "client": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "client auth" + ] + }, + "peer": { + "expiry": "43800h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } +}