From 4b8b52e0de568002420fa38e8dfdb5db4b714bc9 Mon Sep 17 00:00:00 2001 From: Yibo Cai Date: Thu, 7 Nov 2019 17:55:12 +0800 Subject: [PATCH] Support Arm64 image Update CI merge job to build and push Arm64 image to quay.io/cephcsi/cephcsi:version-arm64. Add CI PR job running on Travis Arm64 nodes to make sure cephcsi compiles successfully on Arm64. No CI test job is availabe for Arm64 now due to below issues - k8s-csi sidecar images for Arm64 are not available - Travis Arm64 CI job runs inside unprivileged LXD which blocks launching minikube test environment Signed-off-by: Yibo Cai --- .travis.yml | 12 ++++++++++ Makefile | 17 ++++++++++---- README.md | 4 ++++ deploy.sh | 39 ++++++++++++++++++++++++++++++++- deploy/cephcsi/image/Dockerfile | 6 +++-- scripts/minikube.sh | 5 +++-- 6 files changed, 74 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5f6bdd15..de7d290d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ env: - DEP_VERSION=v0.5.4 before_install: + - mkdir -p $GOPATH/bin - curl https://raw.githubusercontent.com/golang/dep/${DEP_VERSION}/install.sh|sh @@ -76,6 +77,17 @@ jobs: - make image-cephcsi || travis_terminate 1; - scripts/travis-functest.sh v1.17.0 || travis_terminate 1; + - name: cephcsi on Arm64 + arch: arm64 + script: + - scripts/skip-doc-change.sh || travis_terminate 0; + - make image-cephcsi || travis_terminate 1; + # No CI test job is availabe for Arm64 now due to below issues + # - k8s-csi sidecar images for Arm64 are not available + # - Travis Arm64 CI job runs inside unprivileged LXD which blocks + # launching minikube test environment + - travis_terminate 0 # deploy only on x86 + deploy: - provider: script on: # yamllint disable-line rule:truthy diff --git a/Makefile b/Makefile index c8d6e2083..46cacf91d 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ CONTAINER_CMD?=docker CSI_IMAGE_NAME=$(if $(ENV_CSI_IMAGE_NAME),$(ENV_CSI_IMAGE_NAME),quay.io/cephcsi/cephcsi) CSI_IMAGE_VERSION=$(if $(ENV_CSI_IMAGE_VERSION),$(ENV_CSI_IMAGE_VERSION),canary) +CSI_IMAGE=$(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION) $(info cephcsi image settings: $(CSI_IMAGE_NAME) version $(CSI_IMAGE_VERSION)) @@ -31,6 +32,11 @@ LDFLAGS += -X $(GO_PROJECT)/pkg/util.GitCommit=$(GIT_COMMIT) # CSI_IMAGE_VERSION will be considered as the driver version LDFLAGS += -X $(GO_PROJECT)/pkg/util.DriverVersion=$(CSI_IMAGE_VERSION) +# set GOARCH explicitly for cross building, default to native architecture +ifeq ($(origin GOARCH), undefined) +GOARCH := $(shell go env GOARCH) +endif + all: cephcsi test: go-test static-check dep-check @@ -52,15 +58,18 @@ func-test: .PHONY: cephcsi cephcsi: if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '$(LDFLAGS) -extldflags "-static"' -o _output/cephcsi ./cmd/ + CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -a -ldflags '$(LDFLAGS) -extldflags "-static"' -o _output/cephcsi ./cmd/ image-cephcsi: cephcsi cp _output/cephcsi deploy/cephcsi/image/cephcsi - $(CONTAINER_CMD) build -t $(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION) deploy/cephcsi/image + chmod +x deploy/cephcsi/image/cephcsi + $(CONTAINER_CMD) build -t $(CSI_IMAGE) deploy/cephcsi/image push-image-cephcsi: image-cephcsi - $(CONTAINER_CMD) push $(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION) - + $(CONTAINER_CMD) tag $(CSI_IMAGE) $(CSI_IMAGE)-$(GOARCH) + $(CONTAINER_CMD) push $(CSI_IMAGE)-$(GOARCH) + # push amd64 image as default one + if [ $(GOARCH) = amd64 ]; then $(CONTAINER_CMD) push $(CSI_IMAGE); fi clean: go clean -r -x diff --git a/README.md b/README.md index b23c8f059..f5fe4af7e 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,10 @@ Independent CSI plugins are provided to support RBD and CephFS backed volumes, refer [cephfs doc](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md). - For example usage of RBD and CephFS CSI plugins, see examples in `examples/`. +NOTE: + +- Ceph CSI **`Arm64`** support is experimental. + ## Project status Status: **GA** diff --git a/deploy.sh b/deploy.sh index 04985d4c4..918ea1ffc 100755 --- a/deploy.sh +++ b/deploy.sh @@ -33,6 +33,42 @@ push_helm_charts() { } +# Build and push images. Steps as below: +# 1. get base image from original Dockerfile (FROM ceph/ceph:v14.2) +# 2. parse manifest to get image digest per arch (sha256:XXX, sha256:YYY) +# 3. patch Dockerfile with amd64 base image (FROM ceph/ceph:v14.2@sha256:XXX) +# 4. build and push amd64 image +# 5. patch Dockerfile with arm64 base image (FROM ceph/ceph:v14.2@sha256:YYY) +# 6. build and push arm64 image +build_push_images() { + # "docker manifest" requires experimental feature enabled + export DOCKER_CLI_EXPERIMENTAL=enabled + + # get baseimg (ceph/ceph:tag) + dockerfile="deploy/cephcsi/image/Dockerfile" + baseimg=$(awk '/^FROM/ {print $NF}' "${dockerfile}") + + # get image digest per architecture + # { + # "arch": "amd64", + # "digest": "sha256:XXX" + # } + # { + # "arch": "arm64", + # "digest": "sha256:YYY" + # } + manifests=$("${CONTAINER_CMD:-docker}" manifest inspect "${baseimg}" | jq '.manifests[] | {arch: .platform.architecture, digest: .digest}') + + # build and push per arch images + for ARCH in amd64 arm64; do + ifs=$IFS; IFS= + digest=$(awk -v ARCH=${ARCH} '{if (archfound) {print $NF; exit 0}}; {archfound=($0 ~ "arch.*"ARCH)}' <<< "${manifests}") + IFS=$ifs + sed -i "s|\(^FROM.*\)${baseimg}.*$|\1${baseimg}@${digest}|" "${dockerfile}" + GOARCH=${ARCH} make push-image-cephcsi + done +} + if [ "${TRAVIS_BRANCH}" == 'master' ]; then export ENV_CSI_IMAGE_VERSION='canary' else @@ -42,10 +78,11 @@ fi if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then "${CONTAINER_CMD:-docker}" login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io - make push-image-cephcsi set -xe + build_push_images + mkdir -p tmp pushd tmp >/dev/null diff --git a/deploy/cephcsi/image/Dockerfile b/deploy/cephcsi/image/Dockerfile index 9d64f696d..881e2947f 100644 --- a/deploy/cephcsi/image/Dockerfile +++ b/deploy/cephcsi/image/Dockerfile @@ -2,12 +2,14 @@ FROM ceph/ceph:v14.2 LABEL maintainers="Ceph-CSI Authors" LABEL description="Ceph-CSI Plugin" +# To support cross building, do NOT RUN native binaries here. +# If we have to run native binaries, qemu-user-static must be installed on +# build host and mounted to container. + # Removing ceph-iscsi repo to workaround the repo issue while upgrading #RUN rm -f /etc/yum.repos.d/ceph-iscsi.repo && yum -y update && yum clean all ENV CSIBIN=/usr/local/bin/cephcsi COPY cephcsi $CSIBIN -RUN chmod +x $CSIBIN - ENTRYPOINT ["/usr/local/bin/cephcsi"] diff --git a/scripts/minikube.sh b/scripts/minikube.sh index ffae0c377..245d15164 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -44,16 +44,17 @@ function install_minikube() { fi echo "Installing minikube. Version: ${MINIKUBE_VERSION}" - curl -Lo minikube https://storage.googleapis.com/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64 && chmod +x minikube && mv minikube /usr/local/bin/ + curl -Lo minikube https://storage.googleapis.com/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-"${MINIKUBE_ARCH}" && chmod +x minikube && mv minikube /usr/local/bin/ } function install_kubectl() { # Download kubectl, which is a requirement for using minikube. echo "Installing kubectl. Version: ${KUBE_VERSION}" - curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ + curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/"${MINIKUBE_ARCH}"/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/ } # configure minikube +MINIKUBE_ARCH=${MINIKUBE_ARCH:-"amd64"} MINIKUBE_VERSION=${MINIKUBE_VERSION:-"latest"} KUBE_VERSION=${KUBE_VERSION:-"v1.14.2"} MEMORY=${MEMORY:-"3000"}