From f9ffda288b81d6fcb8d0e564c09359b70cedd2a4 Mon Sep 17 00:00:00 2001 From: Hinsteny Hisoka Date: Thu, 8 Jun 2023 11:00:30 +0800 Subject: [PATCH] feature: e2e framework supports testing wasm plugins (#369) --- .github/workflows/build-and-test.yaml | 41 ++++++++++++- .gitignore | 3 +- Makefile.core.mk | 20 +++++++ helm/core/templates/deployment.yaml | 10 ++++ test/README.md | 7 ++- .../conformance/tests/request-block.go | 59 +++++++++++++++++++ .../conformance/tests/request-block.yaml | 45 ++++++++++++++ test/ingress/e2e_test.go | 59 +++++++++++-------- .../{cluster.conf => build-wasm-plugins.sh} | 27 +++------ tools/hack/create-cluster.sh | 42 +++++++++++++ 10 files changed, 266 insertions(+), 47 deletions(-) create mode 100644 test/ingress/conformance/tests/request-block.go create mode 100644 test/ingress/conformance/tests/request-block.yaml rename tools/hack/{cluster.conf => build-wasm-plugins.sh} (61%) mode change 100644 => 100755 diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 5415e426f..754469dfb 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -144,8 +144,45 @@ jobs: - name: "Run Ingress Conformance Tests" run: GOPROXY="https://proxy.golang.org,direct" make ingress-conformance-test - publish: + ingress-wasmplugin-test: runs-on: ubuntu-latest - needs: [ingress-conformance-test,gateway-conformance-test] + needs: [build] + steps: + - uses: actions/checkout@v3 + + - name: "Setup Go" + uses: actions/setup-go@v3 + with: + go-version: 1.19 + + - name: Setup Golang Caches + uses: actions/cache@v3 + with: + path: |- + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ github.run_id }} + restore-keys: | + ${{ runner.os }}-go + + - name: Setup Submodule Caches + uses: actions/cache@v3 + with: + path: |- + envoy + istio + external + .git/modules + key: ${{ runner.os }}-submodules-${{ github.run_id }} + restore-keys: ${{ runner.os }}-submodules + + - run: git stash # restore patch + + - name: "Run Ingress WasmPlugins Tests" + run: GOPROXY="https://proxy.golang.org,direct" make ingress-wasmplugin-test + + publish: + runs-on: ubuntu-latest + needs: [ingress-conformance-test,gateway-conformance-test,ingress-wasmplugin-test] steps: - uses: actions/checkout@v3 diff --git a/.gitignore b/.gitignore index d0280e0b8..88083abf6 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ bazel-out bazel-testlogs bazel-wasm-cpp tools/bin/ -helm/**/charts/**.tgz \ No newline at end of file +helm/**/charts/**.tgz +tools/hack/cluster.conf \ No newline at end of file diff --git a/Makefile.core.mk b/Makefile.core.mk index 556d44222..b68c7f92b 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -128,6 +128,9 @@ build-gateway: prebuild external/package/envoy.tar.gz build-istio: prebuild cd external/istio; rm -rf out; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker +build-wasmplugins: + ./tools/hack/build-wasm-plugins.sh + pre-install: cp api/kubernetes/customresourcedefinitions.gen.yaml helm/core/crds @@ -145,6 +148,9 @@ ISTIO_LATEST_IMAGE_TAG ?= 1.0.0 install-dev: pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' +install-dev-wasmplugin: build-wasmplugins pre-install + helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' --set 'global.volumeWasmPlugins=true' + uninstall: helm uninstall higress -n higress-system @@ -197,6 +203,10 @@ gateway-conformance-test: .PHONY: ingress-conformance-test ingress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev run-ingress-e2e-test delete-cluster +# ingress-wasmplugin-test runs ingress wasmplugin tests. +.PHONY: ingress-wasmplugin-test +ingress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-ingress-e2e-test-wasmplugin delete-cluster + # create-cluster creates a kube cluster with kind. .PHONY: create-cluster create-cluster: $(tools/kind) @@ -221,3 +231,13 @@ run-ingress-e2e-test: @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available go test -v -tags conformance ./test/ingress/e2e_test.go --ingress-class=higress --debug=true + +# run-ingress-e2e-test starts to run ingress e2e tests. +.PHONY: run-ingress-e2e-test-wasmplugin +run-ingress-e2e-test-wasmplugin: + @echo -e "\n\033[36mRunning higress conformance tests...\033[0m" + @echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available + @echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n" + kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available + go test -v -tags conformance ./test/ingress/e2e_test.go -isWasmPluginTest=true --ingress-class=higress --debug=true diff --git a/helm/core/templates/deployment.yaml b/helm/core/templates/deployment.yaml index 2cb9cb593..01deee413 100644 --- a/helm/core/templates/deployment.yaml +++ b/helm/core/templates/deployment.yaml @@ -206,6 +206,10 @@ spec: - mountPath: /etc/istio/custom-bootstrap name: custom-bootstrap-volume {{- end }} + {{- if .Values.global.volumeWasmPlugins }} + - mountPath: /opt/plugins + name: local-wasmplugins-volume + {{- end }} {{- if .Values.gateway.hostNetwork }} hostNetwork: {{ .Values.gateway.hostNetwork }} dnsPolicy: ClusterFirstWithHostNet @@ -274,3 +278,9 @@ spec: containerName: higress-gateway divisor: 1m resource: limits.cpu + {{- if .Values.global.volumeWasmPlugins }} + - name: local-wasmplugins-volume + hostPath: + path: /opt/plugins + type: Directory + {{- end }} diff --git a/test/README.md b/test/README.md index 4d77c1658..1a152fb40 100644 --- a/test/README.md +++ b/test/README.md @@ -15,7 +15,12 @@ Higress e2e tests are mainly focusing on two parts for now: ![ingress-workflow](./ingress/pipeline.png) -Higress provides make target to run ingress api conformance tests: `make ingress-conformance-test`. It can be divided into below steps: +Higress provides make target to run ingress api conformance tests and wasmplugin tests, + ++ API Tests: `make ingress-conformance-test` ++ WasmPlugin Tests: `make ingress-wasmplugin-test` + +It can be divided into below steps: 1. delete-cluster: checks if we have undeleted kind cluster. 2. create-cluster: create a new kind cluster. diff --git a/test/ingress/conformance/tests/request-block.go b/test/ingress/conformance/tests/request-block.go new file mode 100644 index 000000000..c1025e138 --- /dev/null +++ b/test/ingress/conformance/tests/request-block.go @@ -0,0 +1,59 @@ +// Copyright (c) 2022 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "testing" + + "github.com/alibaba/higress/test/ingress/conformance/utils/http" + "github.com/alibaba/higress/test/ingress/conformance/utils/suite" +) + +func init() { + HigressConformanceTests = append(HigressConformanceTests, WasmPluginsRequestBlock) +} + +var WasmPluginsRequestBlock = suite.ConformanceTest{ + ShortName: "WasmPluginsRequestBlock", + Description: "The Ingress in the higress-conformance-infra namespace test the request-block wasmplugins.", + Manifests: []string{"tests/request-block.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + testcases := []http.Assertion{ + { + Meta: http.AssertionMeta{ + TargetBackend: "infra-backend-v1", + TargetNamespace: "higress-conformance-infra", + }, + Request: http.AssertionRequest{ + ActualRequest: http.Request{ + Host: "foo.com", + Path: "/swagger.html", + UnfollowRedirect: true, + }, + }, + Response: http.AssertionResponse{ + ExpectedResponse: http.Response{ + StatusCode: 403, + }, + }, + }, + } + t.Run("WasmPlugins request-block", func(t *testing.T) { + for _, testcase := range testcases { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase) + } + }) + }, +} diff --git a/test/ingress/conformance/tests/request-block.yaml b/test/ingress/conformance/tests/request-block.yaml new file mode 100644 index 000000000..420ae237f --- /dev/null +++ b/test/ingress/conformance/tests/request-block.yaml @@ -0,0 +1,45 @@ +# Copyright (c) 2022 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + nginx.ingress.kubernetes.io/app-root: "/foo" + name: httproute-app-root + namespace: higress-conformance-infra +spec: + ingressClassName: higress + rules: + - host: "foo.com" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: infra-backend-v1 + port: + number: 8080 +--- +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: request-block + namespace: higress-system +spec: + defaultConfig: + block_urls: + - "swagger.html" + url: file:///opt/plugins/wasm-go/extensions/request-block/plugin.wasm diff --git a/test/ingress/e2e_test.go b/test/ingress/e2e_test.go index 920bb4a3a..f0503ad6d 100644 --- a/test/ingress/e2e_test.go +++ b/test/ingress/e2e_test.go @@ -28,6 +28,8 @@ import ( "github.com/alibaba/higress/test/ingress/conformance/utils/suite" ) +var isWasmPluginTest = flag.Bool("isWasmPluginTest", false, "") + func TestHigressConformanceTests(t *testing.T) { flag.Parse() @@ -49,32 +51,39 @@ func TestHigressConformanceTests(t *testing.T) { }) cSuite.Setup(t) + var higressTests []suite.ConformanceTest - higressTests := []suite.ConformanceTest{ - tests.HTTPRouteSimpleSameNamespace, - tests.HTTPRouteHostNameSameNamespace, - tests.HTTPRouteRewritePath, - tests.HTTPRouteRewriteHost, - tests.HTTPRouteCanaryHeader, - tests.HTTPRouteEnableCors, - tests.HTTPRouteEnableIgnoreCase, - tests.HTTPRouteMatchMethods, - tests.HTTPRouteMatchQueryParams, - tests.HTTPRouteMatchHeaders, - tests.HTTPRouteAppRoot, - tests.HTTPRoutePermanentRedirect, - tests.HTTPRoutePermanentRedirectCode, - tests.HTTPRouteTemporalRedirect, - tests.HTTPRouteSameHostAndPath, - tests.HTTPRouteCanaryHeaderWithCustomizedHeader, - tests.HTTPRouteWhitelistSourceRange, - tests.HTTPRouteCanaryWeight, - tests.HTTPRouteMatchPath, - tests.HttpForceRedirectHttps, - tests.HttpRedirectAsHttps, - tests.HTTPRouteRequestHeaderControl, - tests.HTTPRouteDownstreamEncryption, - tests.HTTPRouteFullPathRegex, + if *isWasmPluginTest { + higressTests = []suite.ConformanceTest{ + tests.WasmPluginsRequestBlock, + } + } else { + higressTests = []suite.ConformanceTest{ + tests.HTTPRouteSimpleSameNamespace, + tests.HTTPRouteHostNameSameNamespace, + tests.HTTPRouteRewritePath, + tests.HTTPRouteRewriteHost, + tests.HTTPRouteCanaryHeader, + tests.HTTPRouteEnableCors, + tests.HTTPRouteEnableIgnoreCase, + tests.HTTPRouteMatchMethods, + tests.HTTPRouteMatchQueryParams, + tests.HTTPRouteMatchHeaders, + tests.HTTPRouteAppRoot, + tests.HTTPRoutePermanentRedirect, + tests.HTTPRoutePermanentRedirectCode, + tests.HTTPRouteTemporalRedirect, + tests.HTTPRouteSameHostAndPath, + tests.HTTPRouteCanaryHeaderWithCustomizedHeader, + tests.HTTPRouteWhitelistSourceRange, + tests.HTTPRouteCanaryWeight, + tests.HTTPRouteMatchPath, + tests.HttpForceRedirectHttps, + tests.HttpRedirectAsHttps, + tests.HTTPRouteRequestHeaderControl, + tests.HTTPRouteDownstreamEncryption, + tests.HTTPRouteFullPathRegex, + } } cSuite.Run(t, higressTests) diff --git a/tools/hack/cluster.conf b/tools/hack/build-wasm-plugins.sh old mode 100644 new mode 100755 similarity index 61% rename from tools/hack/cluster.conf rename to tools/hack/build-wasm-plugins.sh index 96225db8a..088574493 --- a/tools/hack/cluster.conf +++ b/tools/hack/build-wasm-plugins.sh @@ -12,21 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# cluster.conf -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - kubeadmConfigPatches: - - | - kind: InitConfiguration - nodeRegistration: - kubeletExtraArgs: - node-labels: "ingress-ready=true" - extraPortMappings: - - containerPort: 80 - hostPort: 80 - protocol: TCP - - containerPort: 443 - hostPort: 443 - protocol: TCP +#!/usr/bin/env bash + +set -euo pipefail + +cd ./plugins/wasm-go/ + +# developer need to declear new wasmplugins here for test +PLUGIN_NAME=http-call make build +PLUGIN_NAME=request-block make build \ No newline at end of file diff --git a/tools/hack/create-cluster.sh b/tools/hack/create-cluster.sh index 13f8881d0..854a523d4 100755 --- a/tools/hack/create-cluster.sh +++ b/tools/hack/create-cluster.sh @@ -20,6 +20,48 @@ set -euo pipefail CLUSTER_NAME=${CLUSTER_NAME:-"higress"} METALLB_VERSION=${METALLB_VERSION:-"v0.13.7"} KIND_NODE_TAG=${KIND_NODE_TAG:-"v1.25.3"} +PROJECT_DIR=$(pwd) + +echo ${KIND_NODE_TAG} +echo ${CLUSTER_NAME} + +cat < "tools/hack/cluster.conf" +# Copyright (c) 2022 Alibaba Group Holding Ltd. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cluster.conf +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + kubeadmConfigPatches: + - | + kind: InitConfiguration + nodeRegistration: + kubeletExtraArgs: + node-labels: "ingress-ready=true" + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP + extraMounts: + - hostPath: ${PROJECT_DIR}/plugins + containerPath: /opt/plugins +EOF ## Create kind cluster. if [[ -z "${KIND_NODE_TAG}" ]]; then