Add support to build multi architecture image

This is back port fo https://github.com/ceph/ceph-csi/pull/866
to release-v2.1 branch

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

.travis.yml

@@ -73,6 +73,10 @@ jobs:
         - make go-test
         - make mod-check || travis_terminate 1;
 
+    - name: Build multi-architecture image for amd64 and arm64
+      script:
+        - travis_wait ./build-multi-arch-image.sh || travis_terminate 1;
+
     - name: containerized build
       script:
         - make containerized-build || travis_terminate 1;
@@ -114,4 +118,4 @@ deploy:
   - provider: script
     on:  # yamllint disable-line rule:truthy
       all_branches: true
-    script: ./deploy.sh
+    script: travis_wait ./deploy.sh

Makefile

@@ -21,8 +21,9 @@ CSI_IMAGE_VERSION=$(if $(ENV_CSI_IMAGE_VERSION),$(ENV_CSI_IMAGE_VERSION),v2.1-ca
 CSI_IMAGE=$(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION)
 
 $(info cephcsi image settings: $(CSI_IMAGE_NAME) version $(CSI_IMAGE_VERSION))
-
+ifndef GIT_COMMIT
 GIT_COMMIT=$(shell git rev-list -1 HEAD)
+endif
 
 GO_PROJECT=github.com/ceph/ceph-csi
 
@@ -33,10 +34,14 @@ LDFLAGS += -X $(GO_PROJECT)/pkg/util.GitCommit=$(GIT_COMMIT)
 LDFLAGS += -X $(GO_PROJECT)/pkg/util.DriverVersion=$(CSI_IMAGE_VERSION)
 
 # set GOARCH explicitly for cross building, default to native architecture
-ifeq ($(origin GOARCH), undefined)
+ifndef GOARCH
 GOARCH := $(shell go env GOARCH)
 endif
 
+ifdef BASE_IMAGE
+BASE_IMAGE_ARG = --build-arg BASE_IMAGE=$(BASE_IMAGE)
+endif
+
 SELINUX := $(shell getenforce 2>/dev/null)
 ifeq ($(SELINUX),Enforcing)
 	SELINUX_VOL_FLAG = :z
@@ -75,10 +80,8 @@ containerized-build: .devel-container-id
 	$(CONTAINER_CMD) build -t $(CSI_IMAGE_NAME):devel -f ./scripts/Dockerfile.devel .
 	$(CONTAINER_CMD) inspect -f '{{.Id}}' $(CSI_IMAGE_NAME):devel > .devel-container-id
 
-image-cephcsi: cephcsi
+image-cephcsi:
-	cp _output/cephcsi deploy/cephcsi/image/cephcsi
+	$(CONTAINER_CMD) build -t $(CSI_IMAGE) -f deploy/cephcsi/image/Dockerfile . --build-arg GOLANG_VERSION=1.13.8 --build-arg CSI_IMAGE_NAME=$(CSI_IMAGE_NAME) --build-arg CSI_IMAGE_VERSION=$(CSI_IMAGE_VERSION) --build-arg GIT_COMMIT=$(GIT_COMMIT) --build-arg GO_ARCH=$(GOARCH) $(BASE_IMAGE_ARG)
-	chmod +x deploy/cephcsi/image/cephcsi
-	$(CONTAINER_CMD) build -t $(CSI_IMAGE) deploy/cephcsi/image
 
 push-image-cephcsi: image-cephcsi
 	$(CONTAINER_CMD) tag $(CSI_IMAGE) $(CSI_IMAGE)-$(GOARCH)

build-multi-arch-image.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -xe
+# "docker manifest" requires experimental feature enabled
+export DOCKER_CLI_EXPERIMENTAL=enabled
+
+# ceph base image used for building multi architecture images
+dockerfile="deploy/cephcsi/image/Dockerfile"
+baseimg=$(awk -F = '/^ARG BASE_IMAGE=/ {print $NF}' "${dockerfile}")
+
+# get image digest per architecture
+# {
+#   "arch": "amd64",
+#   "digest": "sha256:XXX"
+# }
+# {
+#   "arch": "arm64",
+#   "digest": "sha256:YYY"
+# }
+manifests=$(docker manifest inspect "${baseimg}" | jq '.manifests[] | {arch: .platform.architecture, digest: .digest}')
+# qemu-user-static is to enable an execution of different multi-architecture containers by QEMU
+# more info at https://github.com/multiarch/qemu-user-static
+docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
+# build and push per arch images
+for ARCH in amd64 arm64; do
+	ifs=$IFS
+	IFS=
+	digest=$(awk -v ARCH=${ARCH} '{if (archfound) {print $NF; exit 0}}; {archfound=($0 ~ "arch.*"ARCH)}' <<<"${manifests}")
+	IFS=$ifs
+	base_img=${baseimg}@${digest}
+	GOARCH=${ARCH} BASE_IMAGE=${base_img} make image-cephcsi
+done

deploy.sh

@@ -45,9 +45,8 @@ build_push_images() {
 	# "docker manifest" requires experimental feature enabled
 	export DOCKER_CLI_EXPERIMENTAL=enabled
 
-	# get baseimg (ceph/ceph:tag)
 	dockerfile="deploy/cephcsi/image/Dockerfile"
-	baseimg=$(awk '/^FROM/ {print $NF}' "${dockerfile}")
+	baseimg=$(awk -F = '/^ARG BASE_IMAGE=/ {print $NF}' "${dockerfile}")
 
 	# get image digest per architecture
 	# {
@@ -58,16 +57,18 @@ build_push_images() {
 	#   "arch": "arm64",
 	#   "digest": "sha256:YYY"
 	# }
-	manifests=$("${CONTAINER_CMD:-docker}" manifest inspect "${baseimg}" | jq '.manifests[] | {arch: .platform.architecture, digest: .digest}')
+	manifests=$(docker manifest inspect "${baseimg}" | jq '.manifests[] | {arch: .platform.architecture, digest: .digest}')
-
+	# qemu-user-static is to enable an execution of different multi-architecture containers by QEMU
+	# more info at https://github.com/multiarch/qemu-user-static
+	docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
 	# build and push per arch images
 	for ARCH in amd64 arm64; do
 		ifs=$IFS
 		IFS=
 		digest=$(awk -v ARCH=${ARCH} '{if (archfound) {print $NF; exit 0}}; {archfound=($0 ~ "arch.*"ARCH)}' <<<"${manifests}")
 		IFS=$ifs
-		sed -i "s|\(^FROM.*\)${baseimg}.*$|\1${baseimg}@${digest}|" "${dockerfile}"
+		base_image=${baseimg}@${digest}
-		GOARCH=${ARCH} make push-image-cephcsi
+		GOARCH=${ARCH} BASE_IMAGE=${base_image} make push-image-cephcsi
 	done
 }
 

deploy/cephcsi/image/Dockerfile

@@ -1,15 +1,51 @@
-FROM ceph/ceph:v15
+ARG SRC_DIR="/go/src/github.com/ceph/ceph-csi/"
-LABEL maintainers="Ceph-CSI Authors"
+ARG GO_ARCH
-LABEL description="Ceph-CSI Plugin"
+ARG BASE_IMAGE=ceph/ceph:v15
 
-# To support cross building, do NOT RUN native binaries here.
+FROM ${BASE_IMAGE} as builder
-# If we have to run native binaries, qemu-user-static must be installed on
-# build host and mounted to container.
 
-# Removing ceph-iscsi repo to workaround the repo issue while upgrading
+LABEL stage="build"
-#RUN rm -f /etc/yum.repos.d/ceph-iscsi.repo && yum -y update && yum clean all
-ENV CSIBIN=/usr/local/bin/cephcsi
 
-COPY cephcsi $CSIBIN
+ARG GOLANG_VERSION=1.13.9
+ARG CSI_IMAGE_NAME=quay.io/cephcsi/cephcsi
+ARG CSI_IMAGE_VERSION=canary
+ARG GO_ARCH
+ARG SRC_DIR
+ARG GIT_COMMIT
+ARG GOROOT=/usr/local/go
+
+RUN mkdir -p ${GOROOT} && \
+    curl https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${GO_ARCH}.tar.gz | tar xzf - -C ${GOROOT} --strip-components=1
+
+RUN dnf install libcephfs-devel librados-devel librbd-devel /usr/bin/cc make -y
+
+ENV GOROOT=${GOROOT} \
+    GOPATH=/go \
+    CGO_ENABLED=1 \
+    GIT_COMMIT="${GIT_COMMIT}" \
+    ENV_CSI_IMAGE_VERSION="${CSI_IMAGE_VERSION}" \
+    ENV_CSI_IMAGE_NAME="${CSI_IMAGE_NAME}" \
+    PATH="${GOROOT}/bin:${GOPATH}/bin:${PATH}"
+
+
+WORKDIR ${SRC_DIR}
+
+# Copy source directories
+COPY . ${SRC_DIR}
+
+# Build executable
+RUN make cephcsi
+
+#-- Final container
+FROM ${BASE_IMAGE}
+
+ARG SRC_DIR
+
+LABEL maintainers="Ceph-CSI Authors" \
+    version=${CSI_IMAGE_VERSION} \
+    architecture=${GO_ARCH} \
+    description="Ceph-CSI Plugin"
+
+COPY --from=builder ${SRC_DIR}/_output/cephcsi /usr/local/bin/cephcsi
 
 ENTRYPOINT ["/usr/local/bin/cephcsi"]