feature: e2e framework supports testing wasm plugins (#369)

.github/workflows/build-and-test.yaml

@@ -144,8 +144,45 @@ jobs:
     - name: "Run Ingress Conformance Tests"
       run: GOPROXY="https://proxy.golang.org,direct" make ingress-conformance-test
       
-  publish:
+  ingress-wasmplugin-test:
     runs-on: ubuntu-latest
-    needs: [ingress-conformance-test,gateway-conformance-test]
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v3
+      
+    - name: "Setup Go"
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19
+
+    - name: Setup Golang Caches
+      uses: actions/cache@v3
+      with:
+        path: |-
+          ~/.cache/go-build
+          ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ github.run_id }}
+        restore-keys: |
+          ${{ runner.os }}-go
+      
+    - name: Setup Submodule Caches
+      uses: actions/cache@v3
+      with:
+        path: |-
+            envoy
+            istio
+            external
+            .git/modules
+        key: ${{ runner.os }}-submodules-${{ github.run_id }}
+        restore-keys: ${{ runner.os }}-submodules
+          
+    - run: git stash # restore patch
+
+    - name: "Run Ingress WasmPlugins Tests"
+      run: GOPROXY="https://proxy.golang.org,direct" make ingress-wasmplugin-test
+
+  publish:
+    runs-on: ubuntu-latest
+    needs: [ingress-conformance-test,gateway-conformance-test,ingress-wasmplugin-test]
     steps:
     - uses: actions/checkout@v3

.gitignore

@@ -13,3 +13,4 @@ bazel-testlogs
 bazel-wasm-cpp
 tools/bin/
 helm/**/charts/**.tgz
+tools/hack/cluster.conf

Makefile.core.mk

@@ -128,6 +128,9 @@ build-gateway: prebuild external/package/envoy.tar.gz
 build-istio: prebuild
 	cd external/istio; rm -rf out; GOOS_LOCAL=linux TARGET_OS=linux TARGET_ARCH=amd64 BUILD_WITH_CONTAINER=1 DOCKER_BUILD_VARIANTS=default DOCKER_TARGETS="docker.pilot" make docker
 
+build-wasmplugins: 
+	./tools/hack/build-wasm-plugins.sh
+
 pre-install:
 	cp api/kubernetes/customresourcedefinitions.gen.yaml helm/core/crds
 
@@ -145,6 +148,9 @@ ISTIO_LATEST_IMAGE_TAG ?= 1.0.0
 install-dev: pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'
 
+install-dev-wasmplugin: build-wasmplugins pre-install
+	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'  --set 'global.volumeWasmPlugins=true'
+
 uninstall:
 	helm uninstall higress -n higress-system
 
@@ -197,6 +203,10 @@ gateway-conformance-test:
 .PHONY: ingress-conformance-test
 ingress-conformance-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev run-ingress-e2e-test delete-cluster
 
+# ingress-wasmplugin-test runs ingress wasmplugin tests.
+.PHONY: ingress-wasmplugin-test
+ingress-wasmplugin-test: $(tools/kind) delete-cluster create-cluster docker-build kube-load-image install-dev-wasmplugin run-ingress-e2e-test-wasmplugin delete-cluster
+
 # create-cluster creates a kube cluster with kind.
 .PHONY: create-cluster
 create-cluster: $(tools/kind)
@@ -221,3 +231,13 @@ run-ingress-e2e-test:
 	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
 	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
 	go test -v -tags conformance ./test/ingress/e2e_test.go --ingress-class=higress --debug=true
+
+# run-ingress-e2e-test starts to run ingress e2e tests.
+.PHONY: run-ingress-e2e-test-wasmplugin
+run-ingress-e2e-test-wasmplugin:
+	@echo -e "\n\033[36mRunning higress conformance tests...\033[0m"
+	@echo -e "\n\033[36mWaiting higress-controller to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-controller --for=condition=Available
+	@echo -e "\n\033[36mWaiting higress-gateway to be ready...\033[0m\n"
+	kubectl wait --timeout=10m -n higress-system deployment/higress-gateway --for=condition=Available
+	go test -v -tags conformance ./test/ingress/e2e_test.go -isWasmPluginTest=true --ingress-class=higress --debug=true

helm/core/templates/deployment.yaml

@@ -206,6 +206,10 @@ spec:
           - mountPath: /etc/istio/custom-bootstrap
             name: custom-bootstrap-volume
           {{- end }}
+          {{- if .Values.global.volumeWasmPlugins }}
+          - mountPath: /opt/plugins
+            name: local-wasmplugins-volume
+          {{- end }}
       {{- if .Values.gateway.hostNetwork }}
       hostNetwork: {{ .Values.gateway.hostNetwork }}
       dnsPolicy: ClusterFirstWithHostNet
@@ -274,3 +278,9 @@ spec:
               containerName: higress-gateway
               divisor: 1m
               resource: limits.cpu
+      {{- if .Values.global.volumeWasmPlugins }}
+      - name: local-wasmplugins-volume
+        hostPath:
+          path: /opt/plugins
+          type: Directory
+      {{- end }}

test/README.md

@@ -15,7 +15,12 @@ Higress e2e tests are mainly focusing on two parts for now:
 
 ![ingress-workflow](./ingress/pipeline.png)
 
-Higress provides make target to run ingress api conformance tests: `make ingress-conformance-test`. It can be divided into below steps:
+Higress provides make target to run ingress api conformance tests and wasmplugin tests,
+
++  API Tests: `make ingress-conformance-test`
++  WasmPlugin Tests: `make ingress-wasmplugin-test`
+
+It can be divided into below steps:
 
 1. delete-cluster: checks if we have undeleted kind cluster.
 2. create-cluster: create a new kind cluster.

test/ingress/conformance/tests/request-block.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import (
+	"testing"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/http"
+	"github.com/alibaba/higress/test/ingress/conformance/utils/suite"
+)
+
+func init() {
+	HigressConformanceTests = append(HigressConformanceTests, WasmPluginsRequestBlock)
+}
+
+var WasmPluginsRequestBlock = suite.ConformanceTest{
+	ShortName:   "WasmPluginsRequestBlock",
+	Description: "The Ingress in the higress-conformance-infra namespace test the request-block wasmplugins.",
+	Manifests:   []string{"tests/request-block.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		testcases := []http.Assertion{
+			{
+				Meta: http.AssertionMeta{
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:             "foo.com",
+						Path:             "/swagger.html",
+						UnfollowRedirect: true,
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 403,
+					},
+				},
+			},
+		}
+		t.Run("WasmPlugins request-block", func(t *testing.T) {
+			for _, testcase := range testcases {
+				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase)
+			}
+		})
+	},
+}

test/ingress/conformance/tests/request-block.yaml

@@ -0,0 +1,45 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    nginx.ingress.kubernetes.io/app-root: "/foo"
+  name: httproute-app-root
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+  - host: "foo.com"
+    http:
+      paths:
+      - pathType: Prefix
+        path: "/"
+        backend:
+          service:
+            name: infra-backend-v1
+            port:
+              number: 8080
+---
+apiVersion: extensions.higress.io/v1alpha1
+kind: WasmPlugin
+metadata:
+  name: request-block
+  namespace: higress-system
+spec:
+  defaultConfig:
+    block_urls:
+    - "swagger.html"
+  url: file:///opt/plugins/wasm-go/extensions/request-block/plugin.wasm

test/ingress/e2e_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/alibaba/higress/test/ingress/conformance/utils/suite"
 )
 
+var isWasmPluginTest = flag.Bool("isWasmPluginTest", false, "")
+
 func TestHigressConformanceTests(t *testing.T) {
 	flag.Parse()
 
@@ -49,32 +51,39 @@ func TestHigressConformanceTests(t *testing.T) {
 	})
 
 	cSuite.Setup(t)
+	var higressTests []suite.ConformanceTest
 
-	higressTests := []suite.ConformanceTest{
+	if *isWasmPluginTest {
-		tests.HTTPRouteSimpleSameNamespace,
+		higressTests = []suite.ConformanceTest{
-		tests.HTTPRouteHostNameSameNamespace,
+			tests.WasmPluginsRequestBlock,
-		tests.HTTPRouteRewritePath,
+		}
-		tests.HTTPRouteRewriteHost,
+	} else {
-		tests.HTTPRouteCanaryHeader,
+		higressTests = []suite.ConformanceTest{
-		tests.HTTPRouteEnableCors,
+			tests.HTTPRouteSimpleSameNamespace,
-		tests.HTTPRouteEnableIgnoreCase,
+			tests.HTTPRouteHostNameSameNamespace,
-		tests.HTTPRouteMatchMethods,
+			tests.HTTPRouteRewritePath,
-		tests.HTTPRouteMatchQueryParams,
+			tests.HTTPRouteRewriteHost,
-		tests.HTTPRouteMatchHeaders,
+			tests.HTTPRouteCanaryHeader,
-		tests.HTTPRouteAppRoot,
+			tests.HTTPRouteEnableCors,
-		tests.HTTPRoutePermanentRedirect,
+			tests.HTTPRouteEnableIgnoreCase,
-		tests.HTTPRoutePermanentRedirectCode,
+			tests.HTTPRouteMatchMethods,
-		tests.HTTPRouteTemporalRedirect,
+			tests.HTTPRouteMatchQueryParams,
-		tests.HTTPRouteSameHostAndPath,
+			tests.HTTPRouteMatchHeaders,
-		tests.HTTPRouteCanaryHeaderWithCustomizedHeader,
+			tests.HTTPRouteAppRoot,
-		tests.HTTPRouteWhitelistSourceRange,
+			tests.HTTPRoutePermanentRedirect,
-		tests.HTTPRouteCanaryWeight,
+			tests.HTTPRoutePermanentRedirectCode,
-		tests.HTTPRouteMatchPath,
+			tests.HTTPRouteTemporalRedirect,
-		tests.HttpForceRedirectHttps,
+			tests.HTTPRouteSameHostAndPath,
-		tests.HttpRedirectAsHttps,
+			tests.HTTPRouteCanaryHeaderWithCustomizedHeader,
-		tests.HTTPRouteRequestHeaderControl,
+			tests.HTTPRouteWhitelistSourceRange,
-		tests.HTTPRouteDownstreamEncryption,
+			tests.HTTPRouteCanaryWeight,
-		tests.HTTPRouteFullPathRegex,
+			tests.HTTPRouteMatchPath,
+			tests.HttpForceRedirectHttps,
+			tests.HttpRedirectAsHttps,
+			tests.HTTPRouteRequestHeaderControl,
+			tests.HTTPRouteDownstreamEncryption,
+			tests.HTTPRouteFullPathRegex,
+		}
 	}
 
 	cSuite.Run(t, higressTests)

tools/hack/build-wasm-plugins.sh

@@ -12,21 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# cluster.conf
+#!/usr/bin/env bash
-kind: Cluster
+
-apiVersion: kind.x-k8s.io/v1alpha4
+set -euo pipefail
-nodes:
+
-- role: control-plane
+cd ./plugins/wasm-go/
-  kubeadmConfigPatches:
+
-  - |
+# developer need to declear new wasmplugins here for test 
-    kind: InitConfiguration
+PLUGIN_NAME=http-call make build
-    nodeRegistration:
+PLUGIN_NAME=request-block make build
-      kubeletExtraArgs:
-        node-labels: "ingress-ready=true"
-  extraPortMappings:
-  - containerPort: 80
-    hostPort: 80
-    protocol: TCP
-  - containerPort: 443
-    hostPort: 443
-    protocol: TCP

tools/hack/create-cluster.sh

@@ -20,6 +20,48 @@ set -euo pipefail
 CLUSTER_NAME=${CLUSTER_NAME:-"higress"}
 METALLB_VERSION=${METALLB_VERSION:-"v0.13.7"}
 KIND_NODE_TAG=${KIND_NODE_TAG:-"v1.25.3"}
+PROJECT_DIR=$(pwd)
+
+echo ${KIND_NODE_TAG}
+echo ${CLUSTER_NAME}
+
+cat <<EOF > "tools/hack/cluster.conf"
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#      http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cluster.conf
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: InitConfiguration
+    nodeRegistration:
+      kubeletExtraArgs:
+        node-labels: "ingress-ready=true"
+  extraPortMappings:
+  - containerPort: 80
+    hostPort: 80
+    protocol: TCP
+  - containerPort: 443
+    hostPort: 443
+    protocol: TCP
+  extraMounts:
+    - hostPath: ${PROJECT_DIR}/plugins
+      containerPath: /opt/plugins
+EOF
 
 ## Create kind cluster.
 if [[ -z "${KIND_NODE_TAG}" ]]; then