From 86752196220e1935752128cabd4f19f4a19f72bc Mon Sep 17 00:00:00 2001 From: Guillaume Date: Thu, 18 May 2023 19:21:30 +0200 Subject: [PATCH] Add quickstart scripts for easy cluster setup --- .gitignore | 4 ++ clusters/base.yaml | 8 ++- {hosts => hosts.incl}/common.yaml | 0 {hosts => hosts.incl}/master.yaml | 2 +- {hosts => hosts.incl}/node.yaml | 2 +- hosts/.placeholder | 6 -- hosts/m1.yaml | 4 ++ kubeconfig | 19 ++++++ scripts/.common | 107 ++++++++++++++++++++++++++++++ scripts/.template.kubeconfig | 19 ++++++ scripts/0.start_dls.sh | 34 ++++++++++ scripts/1.qemu.sh | 78 ++++++++++++++++++++++ scripts/2.install_addons.sh | 49 ++++++++++++++ 13 files changed, 322 insertions(+), 10 deletions(-) rename {hosts => hosts.incl}/common.yaml (100%) rename {hosts => hosts.incl}/master.yaml (82%) rename {hosts => hosts.incl}/node.yaml (73%) delete mode 100644 hosts/.placeholder create mode 100644 hosts/m1.yaml create mode 100755 kubeconfig create mode 100644 scripts/.common create mode 100644 scripts/.template.kubeconfig create mode 100755 scripts/0.start_dls.sh create mode 100755 scripts/1.qemu.sh create mode 100755 scripts/2.install_addons.sh diff --git a/.gitignore b/.gitignore index 1cd81c5..35b3c7b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ .*sw[po] +cache +data +dist +config.yaml diff --git a/clusters/base.yaml b/clusters/base.yaml index 1bc2923..4547dac 100644 --- a/clusters/base.yaml +++ b/clusters/base.yaml @@ -12,7 +12,8 @@ vars: bootstrap_auths: - name: "my-user" - sshKey: "ssh-ed25519 xxx my-user" +# sshKey: "ssh-ed25519 xxx my-user" + sshKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIomzqVAIqb7BedauhAo2VgbLqme5Jx/vjGUqZLoJqF guillaume@guillaume-archlinux" ssh_keys: - "ssh-ed25519 xxx my-user" @@ -27,7 +28,10 @@ vars: netmask: 24 gateway: 172.17.1.8 dns: - - 172.17.1.1 + #- 172.17.1.1 + - 208.67.220.220 + + dls_base_url: http://172.17.1.8:7606 etcd: image: quay.io/coreos/etcd diff --git a/hosts/common.yaml b/hosts.incl/common.yaml similarity index 100% rename from hosts/common.yaml rename to hosts.incl/common.yaml diff --git a/hosts/master.yaml b/hosts.incl/master.yaml similarity index 82% rename from hosts/master.yaml rename to hosts.incl/master.yaml index ebe85e6..99ca499 100644 --- a/hosts/master.yaml +++ b/hosts.incl/master.yaml @@ -1,4 +1,4 @@ -#!include hosts/common +#!include hosts.incl/common labels: "node-role.novit.io/master": "true" diff --git a/hosts/node.yaml b/hosts.incl/node.yaml similarity index 73% rename from hosts/node.yaml rename to hosts.incl/node.yaml index 3053d45..b195681 100644 --- a/hosts/node.yaml +++ b/hosts.incl/node.yaml @@ -1,4 +1,4 @@ -#!include common +#!include hosts.incl/common labels: "node-role.novit.io/node": "true" diff --git a/hosts/.placeholder b/hosts/.placeholder deleted file mode 100644 index fe658bc..0000000 --- a/hosts/.placeholder +++ /dev/null @@ -1,6 +0,0 @@ -ip: 10.xx.xx.xx -ips: -- 127.0.0.1 -- ::1 -cluster: master -group: master diff --git a/hosts/m1.yaml b/hosts/m1.yaml new file mode 100644 index 0000000..2d7ef3c --- /dev/null +++ b/hosts/m1.yaml @@ -0,0 +1,4 @@ +#!include hosts.incl/master + +ip: 172.17.1.100 +cluster: base diff --git a/kubeconfig b/kubeconfig new file mode 100755 index 0000000..60c0d80 --- /dev/null +++ b/kubeconfig @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + server: https://172.17.1.100:6443 + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUNNakNDQVpPZ0F3SUJBZ0lVZFM5bU5PenhscGJBdXVWcWYyQjIydy96MzdVd0NnWUlLb1pJemowRUF3UXcKTXpFUk1BOEdBMVVFQ2hNSWJtOTJhWFF1YVc4eEhqQWNCZ05WQkFNVEZVUnBjbVZyZEdsc0lFeHZZMkZzSUZObApjblpsY2pBZUZ3MHlNekExTVRneE1UQXpNREJhRncweU9EQTFNVFl4TVRBek1EQmFNRE14RVRBUEJnTlZCQW9UCkNHNXZkbWwwTG1sdk1SNHdIQVlEVlFRREV4VkVhWEpsYTNScGJDQk1iMk5oYkNCVFpYSjJaWEl3Z1pzd0VBWUgKS29aSXpqMENBUVlGSzRFRUFDTURnWVlBQkFBOXdac3h0U2l2RUhUQW50aUx4WFFkUmdxR05sempRMGYwelowQwpyc3hmSGF1K0xrQ1JZNDFoN29ZY21XdVArK09FUlNOL3BGUjhNL3lCZ1dRR0JBbDd2Z0JhZEtpZ01xcHMyalhXCmxUTmt1dnJJbzZhMitESWpjTWNITUhyUUpNUFpzSm8vWGRwOWIyaW5yM2U4dnBiUjFEeEwwRDJMcy9WaWFZL2EKWEE1akR0ZHp2YU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0VHTUE4R0ExVWRFd0VCL3dRRk1BTUJBZjh3SFFZRApWUjBPQkJZRUZIdUpKVjVwN0EzdEhWWDkrb2dkQ1IxZkVFbFBNQW9HQ0NxR1NNNDlCQU1FQTRHTUFEQ0JpQUpDCkFQZHZlOU9nRjFmTjE5T2VjTXc0WEhSVFBGcmMvVHp6SFA0WmlRQVRKdUlLMS9hVEozK1k4QUd5b3ByNElXbmsKUWZZeWtVVmpFREJIVTFLN1JWU1h3YjdJQWtJQnd5K2FORVd6N0hxZE5QQTRKZXV2M3ZKUGFXdU9vRElERWxoMwora3o0ZjFpUkR5QVFKbXZta2dxRzA1M214RmRZU2VIU0NpQ2hhN242RG5kUTgveDYwODA9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + name: localconfig +contexts: +- context: + cluster: localconfig + namespace: kube-system + user: admin + name: localconfig +current-context: localconfig +kind: Config +preferences: {} +users: +- name: admin + user: + token: DU3ITMWLQN2TVGVDTWIURJ42S4 diff --git a/scripts/.common b/scripts/.common new file mode 100644 index 0000000..b3c7842 --- /dev/null +++ b/scripts/.common @@ -0,0 +1,107 @@ +#!/bin/bash + +## Helper funcs + + +pinfo() { + echo -e "\e[32m$@\e[39m" +} + +perror() { + echo -e "\e[31m$@\e[39m" + exit 1 +} + +dls() { + path=$1 + shift + curl -s -H 'Content-Type: application/json' -H 'Authorization: '$DLS_ADM_TOKEN'' http://127.0.0.1:7606${path} "$@" +} + +kctl() { + kubectl --kubeconfig $ctxdir/kubeconfig "$@" +} + +extract_var() { + where=$1 + what=$2 + grep -rh ' $what: ' $ctxdir/$where/*.yaml|awk '{print $2}' +} + +## Run funcs + +prereqs() { + [ $UID != 0 ] && perror This program needs to be run as root. Aborting... + set -ex + cd .. + ctxdir=$PWD +} + +check_conf() { + all_clusters=$(ls $ctxdir/clusters|wc -l) + if [ "$all_clusters" != "1" ]; then + perror "Those helper scripts are not capable of running several clusters at once, check your configuration. Aborting..." + fi +} + +fresh_start() { + rm -rf $ctxdir/secrets +} + +unlock_store() { + # Unlock DLS store after checking if online + # + store_state=$(curl -sw %{http_code} localhost:7606/hosts -o /dev/null) + [ "$?" != 0 ] && perror "Direktil Local Server seems not up, please fix. Aborting." + + if [ "$store_state" == "401" ]; then + pinfo Unlocking the DLS store ... + DLS_ADM_TOKEN=$(dls /public/unlock-store -d "\"${DLS_UNLOCK_TOKEN}\""|tr -d \") + pinfo Admin access token is $DLS_ADM_TOKEN + echo $DLS_ADM_TOKEN > $ctxdir/secrets/.dls_adm_token + chmod 400 > $ctxdir/secrets/.dls_adm_token + fi +} + + +get_hosts() { + hosts_files=$(basename $ctxdir/hosts/*.yaml|sed 's/.yaml//') + for h in ${hosts_files}; do + ip=$(grep ip: $ctxdir/hosts/${h}.yaml|awk '{print $2}') + hosts[$h]="$ip" + done +} + +get_parts() { + for host in ${!hosts[*]}; do + mkdir -p $ctxdir/data/$host + for part in kernel initrd-v2 + do + partfile=$ctxdir/data/$host/$part + test -f $partfile || dls /hosts/$host/$part -o $partfile + done + diskfile=$ctxdir/data/$host/disk.raw + test -f $diskfile || truncate -s ${QEMU_DISK_SIZE:-30G} $diskfile + done +} + + +create_kubeconfig() { + adm_token=$(dls /clusters/base/tokens/admin) + ca_cert=$(dls clusters/base/CAs/cluster/certificate|base64 -w0) + vip=$(extract_var clusters vip) + vip=$(extract_var clusters api_port) + pinfo "Writing new kubeconfig conf in $ctxdir directory, you may want to move it to \~/.kube/ directory for usability" + sed -e "s/__VIP_IP__/$vip/" \ + -e "s/__VIP_API_PORT__/$vip_api_port/" \ + -e "s/__CA_CERT__/$ca_cert/" \ + -e "s/__ADM_TOKEN__/$adm_token/" \ + scripts/.template.kubeconfig > kubeconfig +} + +clean() { + sudo iptables -t nat -D POSTROUTING -j MASQUERADE -s $QEMU_BR_IP \! -o $QEMU_BR_NAME &>/dev/null + sudo iptables -D FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT &>/dev/null + sudo iptables -D FORWARD -j ACCEPT -i $QEMU_BR_NAME &>/dev/null +} + diff --git a/scripts/.template.kubeconfig b/scripts/.template.kubeconfig new file mode 100644 index 0000000..a221570 --- /dev/null +++ b/scripts/.template.kubeconfig @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + server: https://__VIP_IP__:__VIP_API_PORT__ + certificate-authority-data: __CA_CERT__ + name: localconfig +contexts: +- context: + cluster: localconfig + namespace: kube-system + user: admin + name: localconfig +current-context: localconfig +kind: Config +preferences: {} +users: +- name: admin + user: + token: __ADM_TOKEN__ diff --git a/scripts/0.start_dls.sh b/scripts/0.start_dls.sh new file mode 100755 index 0000000..62a0308 --- /dev/null +++ b/scripts/0.start_dls.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +CTR_NAME=dls +# +# +set -xe +IMG=mcluseau/dkl-local-server:beta + +start_store() { + set +e + docker stop $CTR_NAME &>/dev/null + docker wait $CTR_NAME &>/dev/null + set -e + docker run --rm --name $CTR_NAME --net host \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e HTTP_PROXY=$HTTP_PROXY \ + -e HTTPS_PROXY=$HTTPS_PROXY \ + -v $ctxdir:/var/lib/direktil \ + $IMG & +# -auto-unlock 'N0v!T' + sleep 2 +} + + +source .common + +command -v docker 1>/dev/null || perror "Docker is needed, please install it and run again." +systemctl is-active docker &>/dev/null || systemctl start docker + +prereqs +start_store +unlock_store + diff --git a/scripts/1.qemu.sh b/scripts/1.qemu.sh new file mode 100755 index 0000000..cf56f58 --- /dev/null +++ b/scripts/1.qemu.sh @@ -0,0 +1,78 @@ +#! /bin/sh +# +# This collection of scripts aims to install a NOVIT cluster easily, with help of QEMU +# +# Admin token to unlock the DLS store +DLS_UNLOCK_TOKEN=changeme +# +# QEMU local bridge name. If you specificy a custom name, you may have to configure qemu bridge helper to allow it +QEMU_BR_NAME=virbr0 +# +# QEMU VM default disk size +QEMU_DISK_SIZE=30G +# Allocated CPUs to QEMU VMs +QEMU_VM_CPU=4 +# Allocated Memory to QEMU VMs +QEMU_VM_MEM=8096 +################ + + +## QEMU functions +# +prereqs_qemu() { + for com in qemu-system-x86_64 truncate docker iptables ; do + command -v $com 1>/dev/null || error Command $com not found, please install it. Aborting... + done +} +setup_network_qemu() { + # + ip li show $QEMU_BR_NAME &>/dev/null && ip li del $QEMU_BR_NAME + ip li add name $QEMU_BR_NAME type bridge + ip li set $QEMU_BR_NAME up + QEMU_BR_IP=$(extract_var clusters gateway) + QEMU_BR_MASK=$(extract_var clusters netmask) + if [ $(echo $QEMU_BR_IP | wc -w) -gt 1 ]; then + perror "More than one cluster is configured, not compatible with our quick-start setup, exiting" + fi + pinfo "Using detected gateway IP $QEMU_BR_IP for bridge $QEMU_BR_NAME" + ip a add $QEMU_BR_IP dev $QEMU_BR_NAME && { + sudo iptables -t nat -I POSTROUTING -j MASQUERADE -s $QEMU_BR_IP/$QEMU_BR_MASK \! -o $QEMU_BR_NAME + sudo iptables -I FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + sudo iptables -I FORWARD -j ACCEPT -i $QEMU_BR_NAME + } +} +run_qemu() { + id=1 + for host in ${!hosts[*]}; do + ip route add ${hosts[$host]} dev $QEMU_BR_NAME + info "Starting host $host with ip ${hosts[$host]}" + if test -f $ctxdir/data/$host/pid ; then + info "Detected a pid file, killing process in case VM was already started" + kill $(<$ctxdir/$host/pid) && sleep 1 + fi + qemu-system-x86_64 -enable-kvm -smp $QEMU_VM_CPU -m $QEMU_VM_MEM \ + -nic bridge,br=$QEMU_BR_NAME,mac=42:42:42:42:42:0${id} \ + -kernel $ctxdir/data/$host/kernel -initrd $ctxdir/data/$host/initrd-v2 -vga qxl \ + -hda $ctxdir/data/$host/disk.raw & + echo $! >$ctxdir/data/$host/pid + ((++id)) + done + info "$(ls $ctxdir/data/*/pid|wc -w) host(s) have been started" +} +# # # # # # # # + +source .common +prereqs +prereqs_qemu +check_conf +fresh_start +#unlock_store +trap clean SIGINT SIGTERM SIGKILL +declare -A hosts +setup_network_qemu +get_hosts +get_parts +run_qemu +create_kubeconfig +clean + diff --git a/scripts/2.install_addons.sh b/scripts/2.install_addons.sh new file mode 100755 index 0000000..b74db79 --- /dev/null +++ b/scripts/2.install_addons.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Health checking DLS store, and install cluster's addons + +source .common + +prereqs_addons() { + for com in curl kubectl ; do + command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..." + done + cluster=$(basename $ctxdir/clusters/*.yaml|sed 's/.yaml//') +} + +install_addons() { + body='{"Kind":"cluster","Name":"'$cluster'","Assets":["addons"]}' + download_id=$(dls /authorize-download -d "$body"|tr -d \") + dls /public/downloads/${download_id}/addons |\ + kubectl --kubeconfig kubeconfig apply -f - +} + +approve_kubelet_certificates() { + tries=5 + nodes_num=$(kctl get node -oname|wc -l) + while [ "$nodes_num" != "${#hosts[*]}" ] ; do + pinfo "Waiting for certificates requests to be created by Kubelet when it's ready..." + sleep 60s + csrs="$(kctl get csr -oname)" + if [ "$csrs" != "" ]; then + kctl certificate approve $csr + fi + ((tries--)) + if [ $tries < 1 ]; then + pinfo "Timeout waiting for kubelet certificates creation, please investigate why all nodes are not up by now" + break + fi + done + pinfo "All kubelets ($nodes_num) are up, enjoy !" +} + +prereqs +prereqs_addons +check_conf +unlock_store +install_addons + +declare -A hosts +get_hosts # get hosts list +approve_kubelet_certificates # clients and serving certs +