diff --git a/.gitignore b/.gitignore index 35b3c7b..92a31c2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ cache data dist +kubeconfig config.yaml +.dls_adm_token diff --git a/clusters/base.yaml b/clusters/base.yaml index 4547dac..0005f99 100644 --- a/clusters/base.yaml +++ b/clusters/base.yaml @@ -12,8 +12,7 @@ vars: bootstrap_auths: - name: "my-user" -# sshKey: "ssh-ed25519 xxx my-user" - sshKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIomzqVAIqb7BedauhAo2VgbLqme5Jx/vjGUqZLoJqF guillaume@guillaume-archlinux" + sshKey: "ssh-ed25519 xxx my-user" ssh_keys: - "ssh-ed25519 xxx my-user" diff --git a/scripts/.common b/scripts/.common index b3c7842..a750c32 100644 --- a/scripts/.common +++ b/scripts/.common @@ -1,5 +1,9 @@ #!/bin/bash +## Vars + +tknfile=".dls_adm_token" + ## Helper funcs @@ -15,7 +19,10 @@ perror() { dls() { path=$1 shift - curl -s -H 'Content-Type: application/json' -H 'Authorization: '$DLS_ADM_TOKEN'' http://127.0.0.1:7606${path} "$@" + if [ -z "$DLS_ADM_TOKEN" ]; then + test -f $tknfile && DLS_ADM_TOKEN=$(cat $tknfile) + fi + curl -s -H 'Content-Type: application/json' -H 'Authorization: '$DLS_ADM_TOKEN http://127.0.0.1:7606${path} "$@" } kctl() { @@ -25,16 +32,19 @@ kctl() { extract_var() { where=$1 what=$2 - grep -rh ' $what: ' $ctxdir/$where/*.yaml|awk '{print $2}' + grep -rh " $what: " $ctxdir/$where/*.yaml|awk '{print $2}' } ## Run funcs -prereqs() { +check_root() { [ $UID != 0 ] && perror This program needs to be run as root. Aborting... +} + +prereqs() { set -ex - cd .. - ctxdir=$PWD + cd "$(dirname $0)/.." + ctxdir="$PWD" } check_conf() { @@ -51,15 +61,20 @@ fresh_start() { unlock_store() { # Unlock DLS store after checking if online # - store_state=$(curl -sw %{http_code} localhost:7606/hosts -o /dev/null) - [ "$?" != 0 ] && perror "Direktil Local Server seems not up, please fix. Aborting." - - if [ "$store_state" == "401" ]; then - pinfo Unlocking the DLS store ... + if [ -f $tknfile ]; then + DLS_ADM_TOKEN="$(cat $tknfile)" + fi + store_state=$(curl -H 'Content-Type: application/json' -H "Authorization: $DLS_ADM_TOKEN" -sw %{http_code} localhost:7606/hosts -o /dev/null) + if [ "$store_state" == "000" ] ; then + perror "Direktil Local Server seems not up, please fix. Aborting." + elif [ "$store_state" == "200" ] ; then + pinfo "Direktil Local Server store already unlocked" + else + pinfo "Unlocking the DLS store ..." DLS_ADM_TOKEN=$(dls /public/unlock-store -d "\"${DLS_UNLOCK_TOKEN}\""|tr -d \") - pinfo Admin access token is $DLS_ADM_TOKEN - echo $DLS_ADM_TOKEN > $ctxdir/secrets/.dls_adm_token - chmod 400 > $ctxdir/secrets/.dls_adm_token + pinfo "Admin access token is $DLS_ADM_TOKEN" + echo $DLS_ADM_TOKEN > $tknfile + chmod 444 $tknfile fi } @@ -80,26 +95,32 @@ get_parts() { partfile=$ctxdir/data/$host/$part test -f $partfile || dls /hosts/$host/$part -o $partfile done - diskfile=$ctxdir/data/$host/disk.raw + diskfile=$ctxdir/data/$host/disk test -f $diskfile || truncate -s ${QEMU_DISK_SIZE:-30G} $diskfile done } create_kubeconfig() { + if test -f $ctxdir/kubeconfig; then + pinfo "kubeconfig file detected in config dir, won't overwrite... remove it for an update." + return + fi adm_token=$(dls /clusters/base/tokens/admin) - ca_cert=$(dls clusters/base/CAs/cluster/certificate|base64 -w0) - vip=$(extract_var clusters vip) - vip=$(extract_var clusters api_port) + ca_cert=$(dls /clusters/base/CAs/cluster/certificate|base64 -w0) + vip=$(extract_var clusters public_vip) + vip_api_port=$(extract_var clusters api_port) pinfo "Writing new kubeconfig conf in $ctxdir directory, you may want to move it to \~/.kube/ directory for usability" sed -e "s/__VIP_IP__/$vip/" \ -e "s/__VIP_API_PORT__/$vip_api_port/" \ -e "s/__CA_CERT__/$ca_cert/" \ -e "s/__ADM_TOKEN__/$adm_token/" \ - scripts/.template.kubeconfig > kubeconfig + scripts/.template.kubeconfig > $ctxdir/kubeconfig + chmod 444 $ctxdir/kubeconfig } clean() { + set +e sudo iptables -t nat -D POSTROUTING -j MASQUERADE -s $QEMU_BR_IP \! -o $QEMU_BR_NAME &>/dev/null sudo iptables -D FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT &>/dev/null sudo iptables -D FORWARD -j ACCEPT -i $QEMU_BR_NAME &>/dev/null diff --git a/scripts/0.start_dls.sh b/scripts/0.start_dls.sh index 62a0308..ca03825 100755 --- a/scripts/0.start_dls.sh +++ b/scripts/0.start_dls.sh @@ -1,34 +1,49 @@ #!/bin/bash # -CTR_NAME=dls +D2C_CTR_NAME=dir2config +DLS_CTR_NAME=dls +# Admin token to unlock the DLS store +DLS_UNLOCK_TOKEN=changeme # # -set -xe -IMG=mcluseau/dkl-local-server:beta +D2C_IMG=novit.tech/direktil/local-server:b6fa941 +DLS_IMG=novit.tech/direktil/local-server:b6fa941 + +prereqs_dls() { + command -v docker 1>/dev/null || perror "Docker is needed, please install it and run again." + systemctl is-active docker &>/dev/null || systemctl start docker + docker pull $DLS_IMG +} + +dir2config() { + pinfo "Generating config.yaml from Direktil configuration" + docker run --rm --name $D2C_CTR_NAME \ + -v .:/var/lib/direktil -w /var/lib/direktil \ + --entrypoint=/bin/dkl-dir2config \ + $D2C_IMG +} start_store() { - set +e - docker stop $CTR_NAME &>/dev/null - docker wait $CTR_NAME &>/dev/null - set -e - docker run --rm --name $CTR_NAME --net host \ + if docker ps|grep " $DLS_CTR_NAME$" ; then + pinfo "Container $DLS_CTR_NAME seems already running" + return + fi + docker run --rm --name $DLS_CTR_NAME -p 7606:7606 \ -e http_proxy=$http_proxy \ -e https_proxy=$https_proxy \ -e HTTP_PROXY=$HTTP_PROXY \ -e HTTPS_PROXY=$HTTPS_PROXY \ - -v $ctxdir:/var/lib/direktil \ - $IMG & + -v .:/var/lib/direktil \ + $DLS_IMG & # -auto-unlock 'N0v!T' - sleep 2 + sleep 2 } - -source .common - -command -v docker 1>/dev/null || perror "Docker is needed, please install it and run again." -systemctl is-active docker &>/dev/null || systemctl start docker - +source $(dirname $0)/.common +check_root +prereqs_dls prereqs +dir2config start_store unlock_store diff --git a/scripts/1.qemu.sh b/scripts/1.qemu.sh index cf56f58..4ba8fc3 100755 --- a/scripts/1.qemu.sh +++ b/scripts/1.qemu.sh @@ -2,8 +2,6 @@ # # This collection of scripts aims to install a NOVIT cluster easily, with help of QEMU # -# Admin token to unlock the DLS store -DLS_UNLOCK_TOKEN=changeme # # QEMU local bridge name. If you specificy a custom name, you may have to configure qemu bridge helper to allow it QEMU_BR_NAME=virbr0 @@ -25,54 +23,54 @@ prereqs_qemu() { done } setup_network_qemu() { - # - ip li show $QEMU_BR_NAME &>/dev/null && ip li del $QEMU_BR_NAME - ip li add name $QEMU_BR_NAME type bridge - ip li set $QEMU_BR_NAME up + if ! ip li show $QEMU_BR_NAME ; then + ip li add name $QEMU_BR_NAME type bridge + ip li set $QEMU_BR_NAME up + fi QEMU_BR_IP=$(extract_var clusters gateway) QEMU_BR_MASK=$(extract_var clusters netmask) if [ $(echo $QEMU_BR_IP | wc -w) -gt 1 ]; then perror "More than one cluster is configured, not compatible with our quick-start setup, exiting" fi pinfo "Using detected gateway IP $QEMU_BR_IP for bridge $QEMU_BR_NAME" - ip a add $QEMU_BR_IP dev $QEMU_BR_NAME && { + if ! ip a show dev $QEMU_BR_NAME | grep $QEMU_BR_IP ; then + ip a add $QEMU_BR_IP/$QEMU_BR_MASK dev $QEMU_BR_NAME sudo iptables -t nat -I POSTROUTING -j MASQUERADE -s $QEMU_BR_IP/$QEMU_BR_MASK \! -o $QEMU_BR_NAME sudo iptables -I FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT sudo iptables -I FORWARD -j ACCEPT -i $QEMU_BR_NAME - } + fi } run_qemu() { id=1 for host in ${!hosts[*]}; do - ip route add ${hosts[$host]} dev $QEMU_BR_NAME - info "Starting host $host with ip ${hosts[$host]}" + ip route show |grep "${hosts[$host]} dev $QEMU_BR_NAME" ||\ + ip route add ${hosts[$host]} dev $QEMU_BR_NAME if test -f $ctxdir/data/$host/pid ; then - info "Detected a pid file, killing process in case VM was already started" - kill $(<$ctxdir/$host/pid) && sleep 1 + pinfo "Detected a pid file, killing process in case VM was already started" + kill $(cat $ctxdir/data/$host/pid) && sleep 1 fi + pinfo "Starting host $host with ip ${hosts[$host]}" qemu-system-x86_64 -enable-kvm -smp $QEMU_VM_CPU -m $QEMU_VM_MEM \ -nic bridge,br=$QEMU_BR_NAME,mac=42:42:42:42:42:0${id} \ -kernel $ctxdir/data/$host/kernel -initrd $ctxdir/data/$host/initrd-v2 -vga qxl \ - -hda $ctxdir/data/$host/disk.raw & + -drive format=raw,file=$ctxdir/data/$host/disk & echo $! >$ctxdir/data/$host/pid ((++id)) done - info "$(ls $ctxdir/data/*/pid|wc -w) host(s) have been started" + pinfo "$(ls $ctxdir/data/*/pid|wc -w) host(s) have been started" } # # # # # # # # -source .common +source $(dirname $0)/.common +check_root prereqs -prereqs_qemu check_conf -fresh_start -#unlock_store +#fresh_start trap clean SIGINT SIGTERM SIGKILL declare -A hosts setup_network_qemu get_hosts get_parts run_qemu -create_kubeconfig -clean +#clean diff --git a/scripts/2.first_start_k8s.sh b/scripts/2.first_start_k8s.sh new file mode 100755 index 0000000..0e43309 --- /dev/null +++ b/scripts/2.first_start_k8s.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Starting control plane + + +prereqs_control_plane() { + for com in kubectl ; do + command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..." + done +} +prereqs_addons() { + for com in curl kubectl ; do + command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..." + done + cluster=$(basename $ctxdir/clusters/*.yaml|sed 's/.yaml//') +} + +checkup() { + for host in ${!hosts[*]}; do + tries=3 + while : + do + pinfo "Checking availability of node $host..." + ssh root@${hosts[$host]} <<< true &>/dev/null + if [ $? == 0 ]; then + pinfo "VM $host is up!" + break + else + ((tries--)) + if [ $tries < 1 ]; then + pinfo "Timeout waiting for node detection, please investigate why node $host is not up by now" + break + fi + sleep 30 + fi + done + done +} + +start_control_plane() { + for host in ${!hosts[*]}; do + ssh root@${hosts[$host]} << EOF + if ls /etc/kubernetes/manifests.static/* &>/dev/null ; then + mv /etc/kubernetes/manifests.static/* /var/lib/kubelet/manifests/ + fi +EOF + done +} + +install_addons() { + body='{"Kind":"cluster","Name":"'$cluster'","Assets":["addons"]}' + download_id=$(dls /authorize-download -d "$body"|tr -d \") + dls /public/downloads/${download_id}/addons |\ + kctl apply -f - +} + +approve_kubelet_certificates() { + tries=5 + nodes_num=$(kctl get node -oname|wc -l) + while [ "$nodes_num" != "${#hosts[*]}" ] ; do + pinfo "Waiting for certificates requests to be created by Kubelet when it's ready... ($tries/5)" + sleep 60s + csrs="$(kctl get csr|awk '/Pending/ {print $1}')" + if [ "$csrs" != "" ]; then + kctl certificate approve $csrs + fi + ((tries--)) + if [ "$tries" < 1 ]; then + pinfo "Timeout waiting for kubelet certificates creation, please investigate why all nodes are not up by now" + break + fi + done + pinfo "All kubelets ($nodes_num) are up, enjoy !" +} + +source $(dirname $0)/.common +prereqs +prereqs_control_plane +prereqs_addons +check_conf +unlock_store +declare -A hosts && get_hosts +checkup +start_control_plane +create_kubeconfig +install_addons +approve_kubelet_certificates # clients and serving certs + diff --git a/scripts/2.install_addons.sh b/scripts/2.install_addons.sh deleted file mode 100755 index b74db79..0000000 --- a/scripts/2.install_addons.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Health checking DLS store, and install cluster's addons - -source .common - -prereqs_addons() { - for com in curl kubectl ; do - command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..." - done - cluster=$(basename $ctxdir/clusters/*.yaml|sed 's/.yaml//') -} - -install_addons() { - body='{"Kind":"cluster","Name":"'$cluster'","Assets":["addons"]}' - download_id=$(dls /authorize-download -d "$body"|tr -d \") - dls /public/downloads/${download_id}/addons |\ - kubectl --kubeconfig kubeconfig apply -f - -} - -approve_kubelet_certificates() { - tries=5 - nodes_num=$(kctl get node -oname|wc -l) - while [ "$nodes_num" != "${#hosts[*]}" ] ; do - pinfo "Waiting for certificates requests to be created by Kubelet when it's ready..." - sleep 60s - csrs="$(kctl get csr -oname)" - if [ "$csrs" != "" ]; then - kctl certificate approve $csr - fi - ((tries--)) - if [ $tries < 1 ]; then - pinfo "Timeout waiting for kubelet certificates creation, please investigate why all nodes are not up by now" - break - fi - done - pinfo "All kubelets ($nodes_num) are up, enjoy !" -} - -prereqs -prereqs_addons -check_conf -unlock_store -install_addons - -declare -A hosts -get_hosts # get hosts list -approve_kubelet_certificates # clients and serving certs - diff --git a/static-pods/master/etcd.yaml b/static-pods/master/etcd.yaml index 5baca4f..3e756a8 100644 --- a/static-pods/master/etcd.yaml +++ b/static-pods/master/etcd.yaml @@ -40,7 +40,7 @@ spec: - name: ETCD_INITIAL_CLUSTER value: {{ range $i, $host := hosts_by_group "master" }}{{ if gt $i 0 }},{{end}}{{$host.name}}=https://{{$host.ip}}:2380{{end}} - name: ETCD_INITIAL_CLUSTER_STATE - value: {{.etcd.cluster_state}} + value: {{ .vars.etcd.cluster_state }} - name: ETCD_INITIAL_CLUSTER_TOKEN value: '{{ token "etcd-initial-cluster" }}' - name: ETCDCTL_ENDPOINTS