Support Arm64 image

Update CI merge job to build and push Arm64 image to
quay.io/cephcsi/cephcsi:version-arm64.

Add CI PR job running on Travis Arm64 nodes to make sure cephcsi
compiles successfully on Arm64.

No CI test job is availabe for Arm64 now due to below issues
- k8s-csi sidecar images for Arm64 are not available
- Travis Arm64 CI job runs inside unprivileged LXD which blocks
  launching minikube test environment

Signed-off-by: Yibo Cai <yibo.cai@arm.com>

.travis.yml

@@ -32,6 +32,7 @@ env:
     - DEP_VERSION=v0.5.4
 
 before_install:
+  - mkdir -p $GOPATH/bin
   - curl
     https://raw.githubusercontent.com/golang/dep/${DEP_VERSION}/install.sh|sh
 
@@ -76,6 +77,17 @@ jobs:
         - make image-cephcsi || travis_terminate 1;
         - scripts/travis-functest.sh v1.17.0 || travis_terminate 1;
 
+    - name: cephcsi on Arm64
+      arch: arm64
+      script:
+        - scripts/skip-doc-change.sh || travis_terminate 0;
+        - make image-cephcsi || travis_terminate 1;
+        # No CI test job is availabe for Arm64 now due to below issues
+        # - k8s-csi sidecar images for Arm64 are not available
+        # - Travis Arm64 CI job runs inside unprivileged LXD which blocks
+        #   launching minikube test environment
+        - travis_terminate 0    # deploy only on x86
+
 deploy:
   - provider: script
     on:  # yamllint disable-line rule:truthy

Makefile

@@ -18,6 +18,7 @@ CONTAINER_CMD?=docker
 
 CSI_IMAGE_NAME=$(if $(ENV_CSI_IMAGE_NAME),$(ENV_CSI_IMAGE_NAME),quay.io/cephcsi/cephcsi)
 CSI_IMAGE_VERSION=$(if $(ENV_CSI_IMAGE_VERSION),$(ENV_CSI_IMAGE_VERSION),canary)
+CSI_IMAGE=$(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION)
 
 $(info cephcsi image settings: $(CSI_IMAGE_NAME) version $(CSI_IMAGE_VERSION))
 
@@ -31,6 +32,11 @@ LDFLAGS += -X $(GO_PROJECT)/pkg/util.GitCommit=$(GIT_COMMIT)
 # CSI_IMAGE_VERSION will be considered as the driver version
 LDFLAGS += -X $(GO_PROJECT)/pkg/util.DriverVersion=$(CSI_IMAGE_VERSION)
 
+# set GOARCH explicitly for cross building, default to native architecture
+ifeq ($(origin GOARCH), undefined)
+GOARCH := $(shell go env GOARCH)
+endif
+
 all: cephcsi
 
 test: go-test static-check dep-check
@@ -52,15 +58,18 @@ func-test:
 .PHONY: cephcsi
 cephcsi:
 	if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '$(LDFLAGS) -extldflags "-static"' -o  _output/cephcsi ./cmd/
+	CGO_ENABLED=0 GOOS=linux GOARCH=$(GOARCH) go build -a -ldflags '$(LDFLAGS) -extldflags "-static"' -o  _output/cephcsi ./cmd/
 
 image-cephcsi: cephcsi
 	cp _output/cephcsi deploy/cephcsi/image/cephcsi
-	$(CONTAINER_CMD) build -t $(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION) deploy/cephcsi/image
+	chmod +x deploy/cephcsi/image/cephcsi
+	$(CONTAINER_CMD) build -t $(CSI_IMAGE) deploy/cephcsi/image
 
 push-image-cephcsi: image-cephcsi
-	$(CONTAINER_CMD) push $(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION)
-
+	$(CONTAINER_CMD) tag $(CSI_IMAGE) $(CSI_IMAGE)-$(GOARCH)
+	$(CONTAINER_CMD) push $(CSI_IMAGE)-$(GOARCH)
+	# push amd64 image as default one
+	if [ $(GOARCH) = amd64 ]; then $(CONTAINER_CMD) push $(CSI_IMAGE); fi
 
 clean:
 	go clean -r -x

README.md

@@ -37,6 +37,10 @@ Independent CSI plugins are provided to support RBD and CephFS backed volumes,
   refer [cephfs doc](https://github.com/ceph/ceph-csi/blob/master/docs/deploy-cephfs.md).
 - For example usage of RBD and CephFS CSI plugins, see examples in `examples/`.
 
+NOTE:
+
+- Ceph CSI **`Arm64`** support is experimental.
+
 ## Project status
 
 Status: **GA**

deploy.sh

@@ -33,6 +33,42 @@ push_helm_charts() {
 
 }
 
+# Build and push images. Steps as below:
+# 1. get base image from original Dockerfile (FROM ceph/ceph:v14.2)
+# 2. parse manifest to get image digest per arch (sha256:XXX, sha256:YYY)
+# 3. patch Dockerfile with amd64 base image (FROM ceph/ceph:v14.2@sha256:XXX)
+# 4. build and push amd64 image
+# 5. patch Dockerfile with arm64 base image (FROM ceph/ceph:v14.2@sha256:YYY)
+# 6. build and push arm64 image
+build_push_images() {
+	# "docker manifest" requires experimental feature enabled
+	export DOCKER_CLI_EXPERIMENTAL=enabled
+
+	# get baseimg (ceph/ceph:tag)
+	dockerfile="deploy/cephcsi/image/Dockerfile"
+	baseimg=$(awk '/^FROM/ {print $NF}' "${dockerfile}")
+
+	# get image digest per architecture
+	# {
+	#   "arch": "amd64",
+	#   "digest": "sha256:XXX"
+	# }
+	# {
+	#   "arch": "arm64",
+	#   "digest": "sha256:YYY"
+	# }
+	manifests=$("${CONTAINER_CMD:-docker}" manifest inspect "${baseimg}" | jq '.manifests[] | {arch: .platform.architecture, digest: .digest}')
+
+	# build and push per arch images
+	for ARCH in amd64 arm64; do
+		ifs=$IFS; IFS=
+		digest=$(awk -v ARCH=${ARCH} '{if (archfound) {print $NF; exit 0}}; {archfound=($0 ~ "arch.*"ARCH)}' <<< "${manifests}")
+		IFS=$ifs
+		sed -i "s|\(^FROM.*\)${baseimg}.*$|\1${baseimg}@${digest}|" "${dockerfile}"
+		GOARCH=${ARCH} make push-image-cephcsi
+	done
+}
+
 if [ "${TRAVIS_BRANCH}" == 'master' ]; then
 	export ENV_CSI_IMAGE_VERSION='canary'
 else
@@ -42,10 +78,11 @@ fi
 
 if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
 	"${CONTAINER_CMD:-docker}" login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io
-	make push-image-cephcsi
 
 	set -xe
 
+	build_push_images
+
 	mkdir -p tmp
 	pushd tmp >/dev/null
 

deploy/cephcsi/image/Dockerfile

@@ -2,12 +2,14 @@ FROM ceph/ceph:v14.2
 LABEL maintainers="Ceph-CSI Authors"
 LABEL description="Ceph-CSI Plugin"
 
+# To support cross building, do NOT RUN native binaries here.
+# If we have to run native binaries, qemu-user-static must be installed on
+# build host and mounted to container.
+
 # Removing ceph-iscsi repo to workaround the repo issue while upgrading
 #RUN rm -f /etc/yum.repos.d/ceph-iscsi.repo && yum -y update && yum clean all
 ENV CSIBIN=/usr/local/bin/cephcsi
 
 COPY cephcsi $CSIBIN
 
-RUN chmod +x $CSIBIN
-
 ENTRYPOINT ["/usr/local/bin/cephcsi"]

scripts/minikube.sh

@@ -44,16 +44,17 @@ function install_minikube() {
     fi
 
     echo "Installing minikube. Version: ${MINIKUBE_VERSION}"
-    curl -Lo minikube https://storage.googleapis.com/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-amd64 && chmod +x minikube && mv minikube /usr/local/bin/
+    curl -Lo minikube https://storage.googleapis.com/minikube/releases/"${MINIKUBE_VERSION}"/minikube-linux-"${MINIKUBE_ARCH}" && chmod +x minikube && mv minikube /usr/local/bin/
 }
 
 function install_kubectl() {
     # Download kubectl, which is a requirement for using minikube.
     echo "Installing kubectl. Version: ${KUBE_VERSION}"
-    curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/amd64/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/
+    curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/"${KUBE_VERSION}"/bin/linux/"${MINIKUBE_ARCH}"/kubectl && chmod +x kubectl && mv kubectl /usr/local/bin/
 }
 
 # configure minikube
+MINIKUBE_ARCH=${MINIKUBE_ARCH:-"amd64"}
 MINIKUBE_VERSION=${MINIKUBE_VERSION:-"latest"}
 KUBE_VERSION=${KUBE_VERSION:-"v1.14.2"}
 MEMORY=${MEMORY:-"3000"}