Various fixes to scripts and conf

.gitignore

@@ -2,4 +2,6 @@
 cache
 data
 dist
+kubeconfig
 config.yaml
+.dls_adm_token

clusters/base.yaml

@@ -12,8 +12,7 @@ vars:
 
   bootstrap_auths:
   - name: "my-user"
-#    sshKey: "ssh-ed25519 xxx my-user"
+    sshKey: "ssh-ed25519 xxx my-user"
-    sshKey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILIomzqVAIqb7BedauhAo2VgbLqme5Jx/vjGUqZLoJqF guillaume@guillaume-archlinux"
 
   ssh_keys:
   - "ssh-ed25519 xxx my-user"

scripts/.common

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+## Vars
+
+tknfile=".dls_adm_token"
+
 ## Helper funcs
 
 
@@ -15,7 +19,10 @@ perror() {
 dls() {
   path=$1
   shift
-  curl -s -H 'Content-Type: application/json' -H 'Authorization: '$DLS_ADM_TOKEN'' http://127.0.0.1:7606${path} "$@"
+  if [ -z "$DLS_ADM_TOKEN" ]; then
+    test -f $tknfile && DLS_ADM_TOKEN=$(cat $tknfile)
+  fi
+  curl -s -H 'Content-Type: application/json' -H 'Authorization: '$DLS_ADM_TOKEN http://127.0.0.1:7606${path} "$@"
 }
 
 kctl() {
@@ -25,16 +32,19 @@ kctl() {
 extract_var() {
   where=$1
   what=$2
-  grep -rh ' $what: ' $ctxdir/$where/*.yaml|awk '{print $2}'
+  grep -rh " $what: " $ctxdir/$where/*.yaml|awk '{print $2}'
 }
 
 ## Run funcs
 
-prereqs() {
+check_root() {
   [ $UID != 0 ] && perror This program needs to be run as root. Aborting...
+}
+
+prereqs() {
   set -ex
-  cd ..
+  cd "$(dirname $0)/.."
-  ctxdir=$PWD
+  ctxdir="$PWD"
 }
 
 check_conf() {
@@ -51,15 +61,20 @@ fresh_start() {
 unlock_store() {
   # Unlock DLS store after checking if online
   #
-  store_state=$(curl -sw %{http_code} localhost:7606/hosts -o /dev/null)
+  if [ -f $tknfile ]; then
-  [ "$?" != 0 ] && perror "Direktil Local Server seems not up, please fix. Aborting."
+    DLS_ADM_TOKEN="$(cat $tknfile)"
-
+  fi
-  if [ "$store_state" == "401" ]; then
+  store_state=$(curl -H 'Content-Type: application/json' -H "Authorization: $DLS_ADM_TOKEN" -sw %{http_code} localhost:7606/hosts -o /dev/null)
-    pinfo Unlocking the DLS store ...
+  if [ "$store_state" == "000" ] ; then
+    perror "Direktil Local Server seems not up, please fix. Aborting."
+  elif [ "$store_state" == "200" ] ; then 
+    pinfo "Direktil Local Server store already unlocked"
+  else
+    pinfo "Unlocking the DLS store ..."
     DLS_ADM_TOKEN=$(dls /public/unlock-store -d "\"${DLS_UNLOCK_TOKEN}\""|tr -d \")
-    pinfo Admin access token is $DLS_ADM_TOKEN
+    pinfo "Admin access token is $DLS_ADM_TOKEN"
-    echo $DLS_ADM_TOKEN > $ctxdir/secrets/.dls_adm_token
+    echo $DLS_ADM_TOKEN > $tknfile
-    chmod 400 > $ctxdir/secrets/.dls_adm_token
+    chmod 444 $tknfile
   fi
 }
 
@@ -80,26 +95,32 @@ get_parts() {
       partfile=$ctxdir/data/$host/$part
       test -f $partfile || dls /hosts/$host/$part -o $partfile
     done
-    diskfile=$ctxdir/data/$host/disk.raw
+    diskfile=$ctxdir/data/$host/disk
     test -f $diskfile || truncate -s ${QEMU_DISK_SIZE:-30G} $diskfile
   done
 }
 
 
 create_kubeconfig() {
+  if test -f $ctxdir/kubeconfig; then
+    pinfo "kubeconfig file detected in config dir, won't overwrite... remove it for an update."
+    return
+  fi
   adm_token=$(dls /clusters/base/tokens/admin)
-  ca_cert=$(dls clusters/base/CAs/cluster/certificate|base64 -w0)
+  ca_cert=$(dls /clusters/base/CAs/cluster/certificate|base64 -w0)
-  vip=$(extract_var clusters vip)
+  vip=$(extract_var clusters public_vip)
-  vip=$(extract_var clusters api_port)
+  vip_api_port=$(extract_var clusters api_port)
   pinfo "Writing new kubeconfig conf in $ctxdir directory, you may want to move it to \~/.kube/ directory for usability"
   sed -e "s/__VIP_IP__/$vip/" \
       -e "s/__VIP_API_PORT__/$vip_api_port/" \
       -e "s/__CA_CERT__/$ca_cert/" \
       -e "s/__ADM_TOKEN__/$adm_token/" \
-      scripts/.template.kubeconfig > kubeconfig
+      scripts/.template.kubeconfig > $ctxdir/kubeconfig
+  chmod 444 $ctxdir/kubeconfig
 }
 
 clean() {
+  set +e
   sudo iptables -t nat -D POSTROUTING -j MASQUERADE -s $QEMU_BR_IP \! -o $QEMU_BR_NAME &>/dev/null
   sudo iptables -D FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT &>/dev/null
   sudo iptables -D FORWARD -j ACCEPT -i $QEMU_BR_NAME &>/dev/null

scripts/0.start_dls.sh

@@ -1,34 +1,49 @@
 #!/bin/bash
 #
-CTR_NAME=dls
+D2C_CTR_NAME=dir2config
+DLS_CTR_NAME=dls
+# Admin token to unlock the DLS store
+DLS_UNLOCK_TOKEN=changeme
 #
 #
-set -xe
+D2C_IMG=novit.tech/direktil/local-server:b6fa941
-IMG=mcluseau/dkl-local-server:beta
+DLS_IMG=novit.tech/direktil/local-server:b6fa941
+
+prereqs_dls() {
+  command -v docker 1>/dev/null || perror "Docker is needed, please install it and run again."
+  systemctl is-active docker &>/dev/null || systemctl start docker
+  docker pull $DLS_IMG
+}
+
+dir2config() {
+  pinfo "Generating config.yaml from Direktil configuration"
+  docker run --rm --name $D2C_CTR_NAME \
+    -v .:/var/lib/direktil -w /var/lib/direktil \
+    --entrypoint=/bin/dkl-dir2config \
+    $D2C_IMG
+}
 
 start_store() {
-  set +e
+  if docker ps|grep " $DLS_CTR_NAME$" ; then
-  docker stop $CTR_NAME &>/dev/null
+    pinfo "Container $DLS_CTR_NAME seems already running"
-  docker wait $CTR_NAME &>/dev/null
+    return
-  set -e
+  fi
-  docker run --rm --name $CTR_NAME --net host \
+  docker run --rm --name $DLS_CTR_NAME -p 7606:7606 \
       -e http_proxy=$http_proxy \
       -e https_proxy=$https_proxy \
       -e HTTP_PROXY=$HTTP_PROXY \
       -e HTTPS_PROXY=$HTTPS_PROXY \
-      -v $ctxdir:/var/lib/direktil \
+      -v .:/var/lib/direktil \
-      $IMG &
+      $DLS_IMG &
 #      -auto-unlock 'N0v!T'
   sleep 2
 }
 
-
+source $(dirname $0)/.common
-source .common
+check_root
-
+prereqs_dls
-command -v docker 1>/dev/null || perror "Docker is needed, please install it and run again."
-systemctl is-active docker &>/dev/null || systemctl start docker
-
 prereqs
+dir2config
 start_store
 unlock_store
 

scripts/1.qemu.sh

@@ -2,8 +2,6 @@
 #
 # This collection of scripts aims to install a NOVIT cluster easily, with help of QEMU
 #
-# Admin token to unlock the DLS store
-DLS_UNLOCK_TOKEN=changeme
 #
 # QEMU local bridge name. If you specificy a custom name, you may have to configure qemu bridge helper to allow it
 QEMU_BR_NAME=virbr0
@@ -25,54 +23,54 @@ prereqs_qemu() {
   done
 }
 setup_network_qemu() {
-  #
+  if ! ip li show $QEMU_BR_NAME ; then
-  ip li show $QEMU_BR_NAME &>/dev/null && ip li del $QEMU_BR_NAME
     ip li add name $QEMU_BR_NAME type bridge
     ip li set $QEMU_BR_NAME up
+  fi
   QEMU_BR_IP=$(extract_var clusters gateway)
   QEMU_BR_MASK=$(extract_var clusters netmask)
   if [ $(echo $QEMU_BR_IP | wc -w) -gt 1 ]; then
     perror "More than one cluster is configured, not compatible with our quick-start setup, exiting"
   fi
   pinfo "Using detected gateway IP $QEMU_BR_IP for bridge $QEMU_BR_NAME"
-  ip a add $QEMU_BR_IP dev $QEMU_BR_NAME && {
+  if ! ip a show dev $QEMU_BR_NAME | grep $QEMU_BR_IP ; then
+    ip a add $QEMU_BR_IP/$QEMU_BR_MASK dev $QEMU_BR_NAME
     sudo iptables -t nat -I POSTROUTING -j MASQUERADE -s $QEMU_BR_IP/$QEMU_BR_MASK \! -o $QEMU_BR_NAME
     sudo iptables -I FORWARD -o $QEMU_BR_NAME -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
     sudo iptables -I FORWARD -j ACCEPT -i $QEMU_BR_NAME
-  }
+  fi
 }
 run_qemu() {
   id=1
   for host in ${!hosts[*]}; do
+    ip route show |grep "${hosts[$host]} dev $QEMU_BR_NAME" ||\
         ip route add ${hosts[$host]} dev $QEMU_BR_NAME
-    info "Starting host $host with ip ${hosts[$host]}"
     if test -f $ctxdir/data/$host/pid ; then
-      info "Detected a pid file, killing process in case VM was already started"
+      pinfo "Detected a pid file, killing process in case VM was already started"
-      kill $(<$ctxdir/$host/pid) && sleep 1
+      kill $(cat $ctxdir/data/$host/pid) && sleep 1
     fi
+    pinfo "Starting host $host with ip ${hosts[$host]}"
     qemu-system-x86_64 -enable-kvm -smp $QEMU_VM_CPU -m $QEMU_VM_MEM \
       -nic bridge,br=$QEMU_BR_NAME,mac=42:42:42:42:42:0${id} \
       -kernel $ctxdir/data/$host/kernel -initrd $ctxdir/data/$host/initrd-v2 -vga qxl \
-      -hda $ctxdir/data/$host/disk.raw &
+      -drive format=raw,file=$ctxdir/data/$host/disk &
     echo $! >$ctxdir/data/$host/pid
     ((++id))
   done
-  info "$(ls $ctxdir/data/*/pid|wc -w) host(s) have been started"
+  pinfo "$(ls $ctxdir/data/*/pid|wc -w) host(s) have been started"
 }
 # # # # # # # #
 
-source .common
+source $(dirname $0)/.common
+check_root
 prereqs
-prereqs_qemu
 check_conf
-fresh_start
+#fresh_start
-#unlock_store
 trap clean SIGINT SIGTERM SIGKILL
 declare -A hosts
 setup_network_qemu
 get_hosts
 get_parts
 run_qemu
-create_kubeconfig
+#clean
-clean
 

scripts/2.first_start_k8s.sh

@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# Starting control plane
+
+
+prereqs_control_plane() {
+  for com in kubectl ; do
+    command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..."
+  done
+}
+prereqs_addons() {
+  for com in curl kubectl ; do
+    command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..."
+  done
+  cluster=$(basename $ctxdir/clusters/*.yaml|sed 's/.yaml//')
+}
+
+checkup() {
+  for host in ${!hosts[*]}; do
+    tries=3
+    while :
+    do
+      pinfo "Checking availability of node $host..."
+      ssh root@${hosts[$host]} <<< true &>/dev/null
+      if [ $? == 0 ]; then
+        pinfo "VM $host is up!"
+        break
+      else
+        ((tries--))
+        if [ $tries < 1 ]; then
+          pinfo "Timeout waiting for node detection, please investigate why node $host is not up by now"
+          break
+        fi
+        sleep 30
+      fi
+    done
+  done
+}
+
+start_control_plane() {
+  for host in ${!hosts[*]}; do
+    ssh root@${hosts[$host]} << EOF
+      if ls /etc/kubernetes/manifests.static/* &>/dev/null ; then
+        mv /etc/kubernetes/manifests.static/* /var/lib/kubelet/manifests/
+      fi
+EOF
+  done
+}
+
+install_addons() {
+  body='{"Kind":"cluster","Name":"'$cluster'","Assets":["addons"]}'
+  download_id=$(dls /authorize-download -d "$body"|tr -d \")
+  dls /public/downloads/${download_id}/addons |\
+      kctl apply -f -
+}
+
+approve_kubelet_certificates() {
+  tries=5
+  nodes_num=$(kctl get node -oname|wc -l)
+  while [ "$nodes_num" != "${#hosts[*]}" ] ; do
+    pinfo "Waiting for certificates requests to be created by Kubelet when it's ready... ($tries/5)"
+    sleep 60s
+    csrs="$(kctl get csr|awk '/Pending/ {print $1}')"
+    if [ "$csrs" != "" ]; then
+      kctl certificate approve $csrs
+    fi
+    ((tries--))
+    if [ "$tries" < 1 ]; then
+      pinfo "Timeout waiting for kubelet certificates creation, please investigate why all nodes are not up by now"
+      break
+    fi
+  done
+  pinfo "All kubelets ($nodes_num) are up, enjoy !"
+}
+
+source $(dirname $0)/.common
+prereqs
+prereqs_control_plane
+prereqs_addons
+check_conf
+unlock_store
+declare -A hosts && get_hosts
+checkup
+start_control_plane
+create_kubeconfig
+install_addons
+approve_kubelet_certificates # clients and serving certs
+

scripts/2.install_addons.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Health checking DLS store, and install cluster's addons
-
-source .common
-
-prereqs_addons() {
-  for com in curl kubectl ; do
-    command -v $com 1>/dev/null || perror "Command $com is not installed, aborting..."
-  done
-  cluster=$(basename $ctxdir/clusters/*.yaml|sed 's/.yaml//')
-}
-
-install_addons() {
-  body='{"Kind":"cluster","Name":"'$cluster'","Assets":["addons"]}'
-  download_id=$(dls /authorize-download -d "$body"|tr -d \")
-  dls /public/downloads/${download_id}/addons |\
-      kubectl --kubeconfig kubeconfig apply -f -
-}
-
-approve_kubelet_certificates() {
-  tries=5
-  nodes_num=$(kctl get node -oname|wc -l)
-  while [ "$nodes_num" != "${#hosts[*]}" ] ; do
-    pinfo "Waiting for certificates requests to be created by Kubelet when it's ready..."
-    sleep 60s
-    csrs="$(kctl get csr -oname)"
-    if [ "$csrs" != "" ]; then
-      kctl certificate approve $csr
-    fi
-    ((tries--))
-    if [ $tries < 1 ]; then
-      pinfo "Timeout waiting for kubelet certificates creation, please investigate why all nodes are not up by now"
-      break
-    fi
-  done
-  pinfo "All kubelets ($nodes_num) are up, enjoy !"
-}
-
-prereqs
-prereqs_addons
-check_conf
-unlock_store
-install_addons
-
-declare -A hosts
-get_hosts # get hosts list
-approve_kubelet_certificates # clients and serving certs
-

static-pods/master/etcd.yaml

@@ -40,7 +40,7 @@ spec:
           - name:  ETCD_INITIAL_CLUSTER
             value: {{ range $i, $host := hosts_by_group "master" }}{{ if gt $i 0 }},{{end}}{{$host.name}}=https://{{$host.ip}}:2380{{end}}
           - name:  ETCD_INITIAL_CLUSTER_STATE
-            value: {{.etcd.cluster_state}}
+            value: {{ .vars.etcd.cluster_state }}
           - name:  ETCD_INITIAL_CLUSTER_TOKEN
             value: '{{ token "etcd-initial-cluster" }}'
           - name:  ETCDCTL_ENDPOINTS