From 8048436604e495711e7111004a0c06bde0be67e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BE=84=E6=BD=AD?= Date: Mon, 14 Aug 2023 20:41:09 +0800 Subject: [PATCH] support multi arch image (#483) --- .github/workflows/build-image-and-push.yaml | 88 ++++++++++++++++-- Makefile.core.mk | 20 ++-- docker/Dockerfile.higress | 5 +- docker/docker.mk | 7 +- .../istio/20230811-hack-multi-arch.patch | 91 +++++++++++++++++++ 5 files changed, 191 insertions(+), 20 deletions(-) create mode 100644 istio/1.12/patches/istio/20230811-hack-multi-arch.patch diff --git a/.github/workflows/build-image-and-push.yaml b/.github/workflows/build-image-and-push.yaml index d0f2d0585..94cff5fd5 100644 --- a/.github/workflows/build-image-and-push.yaml +++ b/.github/workflows/build-image-and-push.yaml @@ -66,13 +66,17 @@ jobs: - name: Build Docker Image and Push run: | - GOPROXY="https://proxy.golang.org,direct" make docker-build + GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make docker-build + GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make docker-build BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress" + docker push $BUILT_IMAGE:$GITHUB_SHA-amd64 + docker push $BUILT_IMAGE:$GITHUB_SHA-arm64 readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}" for image in ${IMAGES[@]}; do echo "Image: $image" - docker tag $BUILT_IMAGE:$GITHUB_SHA $image - docker push $image + docker manifest rm $image + docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64 + docker manifest push $image done build-pilot-image: @@ -132,13 +136,83 @@ jobs: username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} - - name: Build Docker Image and Push + - name: Build Pilot-Discovery Image and Push run: | - sudo GOPROXY="https://proxy.golang.org,direct" make build-istio + sudo GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make build-istio + sudo GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make build-istio BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/pilot" + docker push $BUILT_IMAGE:$GITHUB_SHA-amd64 + docker push $BUILT_IMAGE:$GITHUB_SHA-arm64 readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}" for image in ${IMAGES[@]}; do echo "Image: $image" - docker tag $BUILT_IMAGE:$GITHUB_SHA $image - docker push $image + docker manifest rm $image + docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64 + docker manifest push $image + done + + + build-gateway-image: + runs-on: ubuntu-latest + environment: + name: image-registry-pilot + env: + GATEWAY_IMAGE_REGISTRY: ${{ vars.IMAGE_REGISTRY || 'higress-registry.cn-hangzhou.cr.aliyuncs.com' }} + GATEWAY_IMAGE_NAME: ${{ vars.GATEWAY_IMAGE_NAME || 'higress/gateway' }} + steps: + - name: "Checkout ${{ github.ref }}" + uses: actions/checkout@v3 + with: + fetch-depth: 1 + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + envoy + istio + external + .git/modules + key: ${{ runner.os }}-submodules-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules + + - name: Calculate Docker metadata + id: docker-meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ env.GATEWAY_IMAGE_REGISTRY }}/${{ env.GATEWAY_IMAGE_NAME }} + tags: | + type=sha + type=ref,event=tag + type=semver,pattern={{version}} + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + + - name: Build Gateway Image and Push + run: | + sudo GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=amd64 make build-gateway + sudo GOPROXY="https://proxy.golang.org,direct" TARGET_ARCH=arm64 make build-gateway + BUILT_IMAGE="higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/proxyv2" + docker push $BUILT_IMAGE:$GITHUB_SHA-amd64 + docker push $BUILT_IMAGE:$GITHUB_SHA-arm64 + readarray -t IMAGES <<< "${{ steps.docker-meta.outputs.tags }}" + for image in ${IMAGES[@]}; do + echo "Image: $image" + docker manifest rm $image + docker manifest create $image $BUILT_IMAGE:$GITHUB_SHA-amd64 $BUILT_IMAGE:$GITHUB_SHA-arm64 + docker manifest push $image done diff --git a/Makefile.core.mk b/Makefile.core.mk index 1bb2ece92..2a35aec21 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -17,6 +17,8 @@ GO ?= go export GOPROXY ?= https://proxy.golang.com.cn,direct +TARGET_ARCH ?= amd64 + GOARCH_LOCAL := $(TARGET_ARCH) GOOS_LOCAL := $(TARGET_OS) RELEASE_LDFLAGS='$(GO_LDFLAGS) -extldflags -static -s -w' @@ -130,14 +132,16 @@ export PARENT_GIT_REVISION:=$(TAG) export ENVOY_TAR_PATH:=/home/package/envoy.tar.gz -external/package/envoy.tar.gz: - cd external/proxy; BUILD_WITH_CONTAINER=1 make test_release +external/package/envoy-$(TARGET_ARCH).tar.gz: +# cd external/proxy; BUILD_WITH_CONTAINER=1 make test_release + cd external/package; wget "https://github.com/alibaba/higress/releases/download/v1.0.0/envoy-$(TARGET_ARCH).tar.gz" -build-gateway: prebuild external/package/envoy.tar.gz - cd external/istio; rm -rf out; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker +build-gateway: prebuild external/package/envoy-$(TARGET_ARCH).tar.gz + cp -f external/package/envoy-$(TARGET_ARCH).tar.gz external/package/envoy.tar.gz + cd external/istio; rm -rf out/linux_$(TARGET_ARCH); GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=$(TARGET_ARCH) BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.proxyv2" make docker build-istio: prebuild - cd external/istio; rm -rf out; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker + cd external/istio; rm -rf out/linux_$(TARGET_ARCH); GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=$(TARGET_ARCH) BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker build-wasmplugins: ./tools/hack/build-wasm-plugins.sh @@ -157,10 +161,10 @@ ENVOY_LATEST_IMAGE_TAG ?= 1.1.0 ISTIO_LATEST_IMAGE_TAG ?= 1.1.1 install-dev: pre-install - helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' + helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)-$(TARGET_ARCH)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' install-dev-wasmplugin: build-wasmplugins pre-install - helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' + helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)-$(TARGET_ARCH)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' uninstall: helm uninstall higress -n higress-system @@ -234,7 +238,7 @@ delete-cluster: $(tools/kind) ## Delete kind cluster. # docker pull registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3:1.0.0-RC3 .PHONY: kube-load-image kube-load-image: $(tools/kind) ## Install the Higress image to a kind cluster using the provided $IMAGE and $TAG. - tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG) + tools/hack/kind-load-image.sh higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/higress $(TAG)-$(TARGET_ARCH) tools/hack/docker-pull-image.sh registry.cn-hangzhou.aliyuncs.com/hinsteny/dubbo-provider-demo 0.0.1 tools/hack/docker-pull-image.sh registry.cn-hangzhou.aliyuncs.com/hinsteny/nacos-standlone-rc3 1.0.0-RC3 tools/hack/docker-pull-image.sh docker.io/hashicorp/consul 1.16.0 diff --git a/docker/Dockerfile.higress b/docker/Dockerfile.higress index d36b59f62..3f940e07f 100644 --- a/docker/Dockerfile.higress +++ b/docker/Dockerfile.higress @@ -6,11 +6,12 @@ ARG BASE_VERSION=latest ARG HUB +ARG TARGETARCH + # The following section is used as base image if BASE_DISTRIBUTION=debug # This base image is provided by istio, see: https://github.com/istio/istio/blob/master/docker/Dockerfile.base -FROM ${HUB}/base:${BASE_VERSION} +FROM ${HUB}/base:${BASE_VERSION}-${TARGETARCH} -ARG TARGETARCH COPY ${TARGETARCH:-amd64}/higress /usr/local/bin/higress USER 1337:1337 diff --git a/docker/docker.mk b/docker/docker.mk index aeb869d63..72a9ed4db 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -12,7 +12,7 @@ ## See the License for the specific language governing permissions and ## limitations under the License. -docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} +docker.higress: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} --build-arg TARGETARCH=$(TARGET_ARCH) docker.higress: $(OUT_LINUX)/higress docker.higress: docker/Dockerfile.higress $(HIGRESS_DOCKER_RULE) @@ -28,11 +28,12 @@ DOCKER_ALL_VARIANTS ?= debug distroless # This can be done with DOCKER_BUILD_VARIANTS="default debug" as well, but at the expense of building twice vs building once and tagging twice INCLUDE_UNTAGGED_DEFAULT ?= false DEFAULT_DISTRIBUTION=debug + ifeq ($(BUILDX_PLATFORM), true) ifeq (docker buildx ls | grep -q container-builder,) docker buildx create --name container-builder --use; endif HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker buildx build --no-cache --platform linux/amd64,linux/arm64 $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile$(suffix $@) . --push ); ) else -HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile$(suffix $@) . ); ) -endif \ No newline at end of file +HIGRESS_DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(HIGRESS_DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./docker/docker-copy.sh $^ $(HIGRESS_DOCKER_BUILD_TOP)/$@ && cd $(HIGRESS_DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT))-$(TARGET_ARCH) -f Dockerfile$(suffix $@) . ); ) +endif diff --git a/istio/1.12/patches/istio/20230811-hack-multi-arch.patch b/istio/1.12/patches/istio/20230811-hack-multi-arch.patch new file mode 100644 index 000000000..e56c08e42 --- /dev/null +++ b/istio/1.12/patches/istio/20230811-hack-multi-arch.patch @@ -0,0 +1,91 @@ +diff -Naur istio/pilot/docker/Dockerfile.pilot istio-new/pilot/docker/Dockerfile.pilot +--- istio/pilot/docker/Dockerfile.pilot 2023-08-11 18:01:44.000000000 +0800 ++++ istio-new/pilot/docker/Dockerfile.pilot 2023-08-11 17:50:02.000000000 +0800 +@@ -6,8 +6,10 @@ + + ARG HUB + ++ARG TARGETARCH ++ + # The following section is used as base image if BASE_DISTRIBUTION=debug +-FROM ${HUB:-gcr.io/istio-release}/base:${BASE_VERSION} as debug ++FROM ${HUB:-gcr.io/istio-release}/base:${BASE_VERSION}-${TARGETARCH} as debug + + # The following section is used as base image if BASE_DISTRIBUTION=distroless + # FROM gcr.io/istio-release/distroless:${BASE_VERSION} as distroless +@@ -16,7 +18,6 @@ + # hadolint ignore=DL3006 + FROM ${BASE_DISTRIBUTION:-debug} + +-ARG TARGETARCH + COPY ${TARGETARCH:-amd64}/pilot-discovery /usr/local/bin/pilot-discovery + + # Copy templates for bootstrap generation. +diff -Naur istio/pilot/docker/Dockerfile.proxyv2 istio-new/pilot/docker/Dockerfile.proxyv2 +--- istio/pilot/docker/Dockerfile.proxyv2 2023-08-11 18:01:43.000000000 +0800 ++++ istio-new/pilot/docker/Dockerfile.proxyv2 2023-08-11 17:57:08.000000000 +0800 +@@ -6,8 +6,10 @@ + + ARG HUB + ++ARG TARGETARCH ++ + # The following section is used as base image if BASE_DISTRIBUTION=debug +-FROM ${HUB:-gcr.io/istio-release}/base:${BASE_VERSION} as debug ++FROM ${HUB:-gcr.io/istio-release}/base:${BASE_VERSION}-${TARGETARCH} as debug + + # The following section is used as base image if BASE_DISTRIBUTION=distroless + # This image is a custom built debian11 distroless image with multiarchitecture support. +@@ -25,13 +27,13 @@ + ARG proxy_version + ARG istio_version + ARG SIDECAR=envoy ++ARG TARGETARCH + + # Copy Envoy bootstrap templates used by pilot-agent + COPY envoy_bootstrap.json /var/lib/istio/envoy/envoy_bootstrap_tmpl.json + COPY gcp_envoy_bootstrap.json /var/lib/istio/envoy/gcp_envoy_bootstrap_tmpl.json + + # Install Envoy. +-ARG TARGETARCH + COPY ${TARGETARCH:-amd64}/${SIDECAR} /usr/local/bin/${SIDECAR} + + # Environment variable indicating the exact proxy sha - for debugging or version-specific configs +@@ -39,7 +41,6 @@ + # Environment variable indicating the exact build, for debugging + ENV ISTIO_META_ISTIO_VERSION $istio_version + +-ARG TARGETARCH + COPY ${TARGETARCH:-amd64}/pilot-agent /usr/local/bin/pilot-agent + + # COPY stats-filter.wasm /etc/istio/extensions/stats-filter.wasm +diff -Naur istio/tools/istio-docker.mk istio-new/tools/istio-docker.mk +--- istio/tools/istio-docker.mk 2023-08-11 18:01:44.000000000 +0800 ++++ istio-new/tools/istio-docker.mk 2023-08-11 17:50:02.000000000 +0800 +@@ -85,7 +85,7 @@ + + # Default proxy image. + docker.proxyv2: BUILD_PRE=&& chmod 644 envoy_bootstrap.json gcp_envoy_bootstrap.json +-docker.proxyv2: BUILD_ARGS=--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} --build-arg istio_version=${VERSION} --build-arg BASE_VERSION=${BASE_VERSION} --build-arg SIDECAR=${SIDECAR} --build-arg HUB=${HUB} ++docker.proxyv2: BUILD_ARGS=--build-arg proxy_version=istio-proxy:${PROXY_REPO_SHA} --build-arg istio_version=${VERSION} --build-arg BASE_VERSION=${BASE_VERSION} --build-arg SIDECAR=${SIDECAR} --build-arg HUB=${HUB} --build-arg TARGETARCH=${TARGET_ARCH} + docker.proxyv2: ${ISTIO_ENVOY_BOOTSTRAP_CONFIG_DIR}/envoy_bootstrap.json + docker.proxyv2: ${ISTIO_ENVOY_BOOTSTRAP_CONFIG_DIR}/gcp_envoy_bootstrap.json + docker.proxyv2: $(ISTIO_ENVOY_LINUX_RELEASE_DIR)/${SIDECAR} +@@ -98,7 +98,7 @@ + $(DOCKER_RULE) + + docker.pilot: BUILD_PRE=&& chmod 644 envoy_bootstrap.json gcp_envoy_bootstrap.json +-docker.pilot: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} ++docker.pilot: BUILD_ARGS=--build-arg BASE_VERSION=${BASE_VERSION} --build-arg HUB=${HUB} --build-arg TARGETARCH=${TARGET_ARCH} + docker.pilot: ${ISTIO_ENVOY_BOOTSTRAP_CONFIG_DIR}/envoy_bootstrap.json + docker.pilot: ${ISTIO_ENVOY_BOOTSTRAP_CONFIG_DIR}/gcp_envoy_bootstrap.json + docker.pilot: ${ISTIO_ENVOY_BOOTSTRAP_CONFIG_DIR}/higress-pilot-start.sh +@@ -324,7 +324,7 @@ + # This can be done with DOCKER_BUILD_VARIANTS="default debug" as well, but at the expense of building twice vs building once and tagging twice + INCLUDE_UNTAGGED_DEFAULT ?= false + DEFAULT_DISTRIBUTION=debug +-DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./tools/docker-copy.sh $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT)) -f Dockerfile$(suffix $@) . ); ) ++DOCKER_RULE ?= $(foreach VARIANT,$(DOCKER_BUILD_VARIANTS), time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && TARGET_ARCH=$(TARGET_ARCH) ./tools/docker-copy.sh $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ $(BUILD_PRE) && docker build $(BUILD_ARGS) --build-arg BASE_DISTRIBUTION=$(call normalize-tag,$(VARIANT)) -t $(HUB)/$(subst docker.,,$@):$(TAG)$(call variant-tag,$(VARIANT))-$(TARGET_ARCH) -f Dockerfile$(suffix $@) . ); ) + RENAME_TEMPLATE ?= mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp $(ECHO_DOCKER)/$(VM_OS_DOCKERFILE_TEMPLATE) $(DOCKER_BUILD_TOP)/$@/Dockerfile$(suffix $@) + + # This target will package all docker images used in test and release, without re-building