feat: Support pushing multi-arch images to a custom image registry (#1815)

.github/workflows/build-image-and-push.yaml

@@ -65,12 +65,16 @@ jobs:
 
       - name: Build Docker Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" make docker-buildx-push
+          BUILT_IMAGE=""
-          BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress"
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
+            if [ "$BUILT_IMAGE" == "" ]; then
+              GOPROXY="https://proxy.golang.org,direct" IMG_URL="$image" make docker-buildx-push
+              BUILT_IMAGE="$image"
+            else
+              docker buildx imagetools create $BUILT_IMAGE --tag $image
+            fi
           done
 
   build-pilot-image:
@@ -110,8 +114,13 @@ jobs:
           key: ${{ runner.os }}-go-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-go
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
         uses: actions/cache@v2
@@ -142,19 +151,26 @@ jobs:
 
       - name: Build Pilot-Discovery Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" make build-istio
+          BUILT_IMAGE=""
-          BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/pilot"
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
+            if [ "$BUILT_IMAGE" == "" ]; then
+              TAG=${image#*:}
+              HUB=${image%:*}
+              HUB=${HUB%/*}
+              BUILT_IMAGE="$HUB/pilot:$TAG"
+              GOPROXY="https://proxy.golang.org,direct" IMG_URL="$BUILT_IMAGE" make build-istio
+            fi
+            if [ "$BUILT_IMAGE" != "$image" ]; then
+              docker buildx imagetools create $BUILT_IMAGE --tag $image
+            fi
           done
 
-
   build-gateway-image:
     runs-on: ubuntu-latest
     environment:
-      name: image-registry-pilot
+      name: image-registry-gateway
     env:
       GATEWAY_IMAGE_REGISTRY: ${{ vars.IMAGE_REGISTRY || 'higress-registry.cn-hangzhou.cr.aliyuncs.com' }}
       GATEWAY_IMAGE_NAME: ${{ vars.GATEWAY_IMAGE_NAME || 'higress/gateway' }}
@@ -188,8 +204,13 @@ jobs:
           key: ${{ runner.os }}-go-${{ github.run_id }}
           restore-keys: ${{ runner.os }}-go
 
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          image: tonistiigi/binfmt:qemu-v7.0.0
+
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Cache Docker layers
         uses: actions/cache@v2
@@ -220,10 +241,18 @@ jobs:
 
       - name: Build Gateway Image and Push
         run: |
-          GOPROXY="https://proxy.golang.org,direct" make build-gateway
+          BUILT_IMAGE=""
-          BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/proxyv2"
           readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}"
           for image in ${IMAGES[@]}; do
             echo "Image: $image"
-            docker buildx imagetools create $BUILT_IMAGE:$GITHUB_SHA --tag $image
+            if [ "$BUILT_IMAGE" == "" ]; then
+              TAG=${image#*:}
+              HUB=${image%:*}
+              HUB=${HUB%/*}
+              BUILT_IMAGE="$HUB/proxyv2:$TAG"
+              GOPROXY="https://proxy.golang.org,direct" IMG_URL="$BUILT_IMAGE" make build-gateway
+            fi
+            if [ "$BUILT_IMAGE" != "$image" ]; then
+              docker buildx imagetools create $BUILT_IMAGE --tag $image
+            fi
           done

Makefile.core.mk

@@ -162,13 +162,13 @@ buildx-prepare:
 build-gateway: prebuild buildx-prepare
 	USE_REAL_USER=1 TARGET_ARCH=amd64 DOCKER_TARGETS="docker.proxyv2" ./tools/hack/build-istio-image.sh init
 	USE_REAL_USER=1 TARGET_ARCH=arm64 DOCKER_TARGETS="docker.proxyv2" ./tools/hack/build-istio-image.sh init
-	DOCKER_TARGETS="docker.proxyv2" ./tools/hack/build-istio-image.sh docker.buildx
+	DOCKER_TARGETS="docker.proxyv2" IMG_URL="${IMG_URL}" ./tools/hack/build-istio-image.sh docker.buildx
 
 build-gateway-local: prebuild
 	TARGET_ARCH=${TARGET_ARCH} DOCKER_TARGETS="docker.proxyv2" ./tools/hack/build-istio-image.sh docker
 
 build-istio: prebuild buildx-prepare
-	DOCKER_TARGETS="docker.pilot" ./tools/hack/build-istio-image.sh docker.buildx
+	DOCKER_TARGETS="docker.pilot" IMG_URL="${IMG_URL}" ./tools/hack/build-istio-image.sh docker.buildx
 
 build-istio-local: prebuild
 	TARGET_ARCH=${TARGET_ARCH} DOCKER_TARGETS="docker.pilot" ./tools/hack/build-istio-image.sh docker

docker/docker.mk

@@ -35,6 +35,8 @@ DOCKER_ALL_VARIANTS ?= debug distroless
 INCLUDE_UNTAGGED_DEFAULT ?= false
 DEFAULT_DISTRIBUTION=debug
 
-HIGRESS_DOCKER_BUILDX_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker buildx create --name higress --node higress0 --platform linux/amd64,linux/arm64 --use && docker buildx build --no-cache --platform linux/amd64,linux/arm64 $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/higress:$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . --push  ); )
+IMG ?= higress
-HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/higress:$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . ); )
+IMG_URL ?= $(HUB)/$(IMG):$(TAG)
 
+HIGRESS_DOCKER_BUILDX_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker buildx create --name higress --node higress0 --platform linux/amd64,linux/arm64 --use && docker buildx build --no-cache --platform linux/amd64,linux/arm64 $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(IMG_URL)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . --push  ); )
+HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(IMG_URL)$(call variant-tag,$(VARIANT)) -f Dockerfile.higress . ); )

tools/hack/build-envoy.sh

@@ -30,9 +30,11 @@ fi
 CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${ROOT}/external/package,destination=/home/package "
 CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${ROOT}/external/envoy,destination=/home/envoy "
 
+BUILD_TOOLS_IMG=${BUILD_TOOLS_IMG:-"higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613"}
+
 BUILD_WITH_CONTAINER=1 \
     CONDITIONAL_HOST_MOUNTS=${CONDITIONAL_HOST_MOUNTS} \
     BUILD_ENVOY_BINARY_ONLY=1 \
     DOCKER_RUN_OPTIONS="--user root -e HTTP_PROXY -e HTTPS_PROXY" \
-    IMG=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools-proxy:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613 \
+    IMG=${BUILD_TOOLS_IMG} \
     make test_release

tools/hack/build-istio-image.sh

@@ -25,14 +25,34 @@ CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${ROOT}/external/package,dest
 
 DOCKER_RUN_OPTIONS+="-e HTTP_PROXY -e HTTPS_PROXY"
 
+BUILD_TOOLS_IMG=${BUILD_TOOLS_IMG:-"higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613"}
+
+ORIGINAL_HUB=${HUB}
+
+echo "IMG_URL=$IMG_URL"
+
+if [ -n "$IMG_URL" ]; then
+  TAG=${IMG_URL#*:}
+  HUB=${IMG_URL%:*}
+  HUB=${HUB%/*}
+  if [ "$TAG" == "${IMG_URL}" ]; then
+    TAG=latest
+  fi
+fi
+
+echo "HUB=$HUB"
+echo "TAG=$TAG"
+
 GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=${TARGET_ARCH} \
     ISTIO_ENVOY_LINUX_RELEASE_URL=${ISTIO_ENVOY_LINUX_RELEASE_URL} \
     BUILD_WITH_CONTAINER=1 \
     USE_REAL_USER=${USE_REAL_USER:-0} \
     CONDITIONAL_HOST_MOUNTS=${CONDITIONAL_HOST_MOUNTS} \
     DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="${DOCKER_TARGETS}" \
-    ISTIO_BASE_REGISTRY="${HUB}" \
+    ISTIO_BASE_REGISTRY="${ORIGINAL_HUB}" \
     BASE_VERSION="${HIGRESS_BASE_VERSION}" \
     DOCKER_RUN_OPTIONS=${DOCKER_RUN_OPTIONS} \
-    IMG=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613 \
+    HUB="${HUB}" \
+    TAG="${TAG}" \
+    IMG=${BUILD_TOOLS_IMG} \
     make "$@"

tools/hack/build-istio-pilot.sh

@@ -21,6 +21,8 @@ source "$(dirname -- "$0")/setup-istio-env.sh"
 cd ${ROOT}/external/istio
 rm -rf out/linux_${TARGET_ARCH};
 
+BUILD_TOOLS_IMG=${BUILD_TOOLS_IMG:-"higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613"}
+
 GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=${TARGET_ARCH} \
     ISTIO_ENVOY_LINUX_RELEASE_URL=${ISTIO_ENVOY_LINUX_RELEASE_URL} \
     BUILD_WITH_CONTAINER=1 \
@@ -28,5 +30,5 @@ GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=${TARGET_ARCH} \
     ISTIO_BASE_REGISTRY="${HUB}" \
     BASE_VERSION="${HIGRESS_BASE_VERSION}" \
     DOCKER_RUN_OPTIONS="--user root -e HTTP_PROXY -e HTTPS_PROXY" \
-    IMG=higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/build-tools:release-1.19-ef344298e65eeb2d9e2d07b87eb4e715c2def613 \
+    IMG=${BUILD_TOOLS_IMG} \
     make build-linux