#!/bin/bash -e #Based on ideas from https://github.com/rook/rook/blob/master/tests/scripts/helm.sh TEMP="/tmp/cephcsi-helm-test" HELM="helm" HELM_VERSION=${HELM_VERSION:-"v2.16.5"} arch="${ARCH:-}" CEPHFS_CHART_NAME="ceph-csi-cephfs" RBD_CHART_NAME="ceph-csi-rbd" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DEPLOY_TIMEOUT=600 # ceph-csi specific variables NODE_LABEL_REGION="test.failure-domain/region" NODE_LABEL_ZONE="test.failure-domain/zone" REGION_VALUE="testregion" ZONE_VALUE="testzone" function check_deployment_status() { LABEL=$1 NAMESPACE=$2 echo "Checking Deployment status for label $LABEL in Namespace $NAMESPACE" for ((retry = 0; retry <= DEPLOY_TIMEOUT; retry = retry + 5)); do total_replicas=$(kubectl get deployment -l "$LABEL" -n "$NAMESPACE" -o jsonpath='{.items[0].status.replicas}') ready_replicas=$(kubectl get deployment -l "$LABEL" -n "$NAMESPACE" -o jsonpath='{.items[0].status.readyReplicas}') if [ "$total_replicas" != "$ready_replicas" ]; then echo "Total replicas $total_replicas is not equal to ready count $ready_replicas" kubectl get deployment -l "$LABEL" -n "$NAMESPACE" sleep 10 else echo "Total replicas $total_replicas is equal to ready count $ready_replicas" break fi done if [ "$retry" -gt "$DEPLOY_TIMEOUT" ]; then echo "[Timeout] Failed to get deployment" exit 1 fi } function check_daemonset_status() { LABEL=$1 NAMESPACE=$2 echo "Checking Daemonset status for label $LABEL in Namespace $NAMESPACE" for ((retry = 0; retry <= DEPLOY_TIMEOUT; retry = retry + 5)); do total_replicas=$(kubectl get daemonset -l "$LABEL" -n "$NAMESPACE" -o jsonpath='{.items[0].status.numberAvailable}') ready_replicas=$(kubectl get daemonset -l "$LABEL" -n "$NAMESPACE" -o jsonpath='{.items[0].status.numberReady}') if [ "$total_replicas" != "$ready_replicas" ]; then echo "Total replicas $total_replicas is not equal to ready count $ready_replicas" kubectl get daemonset -l "$LABEL" -n "$NAMESPACE" sleep 10 else echo "Total replicas $total_replicas is equal to ready count $ready_replicas" break fi done if [ "$retry" -gt "$DEPLOY_TIMEOUT" ]; then echo "[Timeout] Failed to get daemonset" exit 1 fi } detectArch() { case "$(uname -m)" in "x86_64" | "amd64") arch="amd64" ;; "aarch64") arch="arm64" ;; "i386") arch="i386" ;; *) echo "Couldn't translate 'uname -m' output to an available arch." echo "Try setting ARCH environment variable to your system arch:" echo "amd64, x86_64. aarch64, i386" exit 1 ;; esac } install() { if ! helm_loc="$(type -p "helm")" || [[ -z ${helm_loc} ]]; then # Download and unpack helm local dist dist="$(uname -s)" mkdir -p ${TEMP} # shellcheck disable=SC2021 dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]") wget "https://storage.googleapis.com/kubernetes-helm/helm-${HELM_VERSION}-${dist}-${arch}.tar.gz" -O "${TEMP}/helm.tar.gz" tar -C "${TEMP}" -zxvf "${TEMP}/helm.tar.gz" fi # set up RBAC for helm kubectl --namespace kube-system create sa tiller kubectl create clusterrolebinding tiller --clusterrole cluster-admin --serviceaccount=kube-system:tiller # Init helm "${HELM}" init --service-account tiller --output yaml | sed 's@apiVersion: extensions/v1beta1@apiVersion: apps/v1@' | sed 's@strategy: {}@selector: {"matchLabels": {"app": "helm", "name": "tiller"}}@' | kubectl apply -f - kubectl -n kube-system patch deploy/tiller-deploy -p '{"spec": {"template": {"spec": {"serviceAccountName": "tiller"}}}}' sleep 5 helm_ready=$(kubectl get pods -l app=helm -n kube-system -o jsonpath='{.items[0].status.containerStatuses[0].ready}') INC=0 until [[ "${helm_ready}" == "true" || $INC -gt 20 ]]; do sleep 10 ((++INC)) helm_ready=$(kubectl get pods -l app=helm -n kube-system -o jsonpath='{.items[0].status.containerStatuses[0].ready}') echo "helm pod status: ${helm_ready}" done if [ "${helm_ready}" != "true" ]; then echo "Helm init not successful" kubectl get pods -l app=helm -n kube-system kubectl logs -lapp=helm --all-containers=true -nkube-system exit 1 fi echo "Helm init successful" } install_cephcsi_helm_charts() { NAMESPACE=$1 if [ -z "$NAMESPACE" ]; then NAMESPACE="default" fi # label the nodes uniformly for domain information for node in $(kubectl get node -o jsonpath='{.items[*].metadata.name}'); do kubectl label node/"${node}" ${NODE_LABEL_REGION}=${REGION_VALUE} kubectl label node/"${node}" ${NODE_LABEL_ZONE}=${ZONE_VALUE} done # install ceph-csi-cephfs and ceph-csi-rbd charts "${HELM}" install "${SCRIPT_DIR}"/../charts/ceph-csi-cephfs --name ${CEPHFS_CHART_NAME} --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-cephfsplugin-provisioner --set nodeplugin.fullnameOverride=csi-cephfsplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true check_deployment_status app=ceph-csi-cephfs ${NAMESPACE} check_daemonset_status app=ceph-csi-cephfs ${NAMESPACE} # deleting configmap as a workaround to avoid configmap already present # issue when installing ceph-csi-rbd kubectl delete cm ceph-csi-config --namespace ${NAMESPACE} "${HELM}" install "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --name ${RBD_CHART_NAME} --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" check_deployment_status app=ceph-csi-rbd ${NAMESPACE} check_daemonset_status app=ceph-csi-rbd ${NAMESPACE} } cleanup_cephcsi_helm_charts() { "${HELM}" del --purge ${CEPHFS_CHART_NAME} "${HELM}" del --purge ${RBD_CHART_NAME} # remove set labels for node in $(kubectl get node --no-headers | cut -f 1 -d ' '); do kubectl label node/"$node" test.failure-domain/region- kubectl label node/"$node" test.failure-domain/zone- done # TODO/LATER we could remove the CSI labels that would have been set as well } helm_reset() { "${HELM}" reset # shellcheck disable=SC2021 rm -rf "${TEMP}" kubectl --namespace kube-system delete sa tiller kubectl delete clusterrolebinding tiller } if [ -z "${arch}" ]; then detectArch fi if ! helm_loc="$(type -p "helm")" || [[ -z ${helm_loc} ]]; then dist="$(uname -s)" # shellcheck disable=SC2021 dist=$(echo "${dist}" | tr "[A-Z]" "[a-z]") HELM="${TEMP}/${dist}-${arch}/helm" fi case "${1:-}" in up) install ;; clean) helm_reset ;; install-cephcsi) install_cephcsi_helm_charts "$2" ;; cleanup-cephcsi) cleanup_cephcsi_helm_charts ;; *) echo "usage:" >&2 echo " $0 up" >&2 echo " $0 clean" >&2 echo " $0 install-cephcsi" >&2 echo " $0 cleanup-cephcsi" >&2 ;; esac