feat: add support for ingress e2e test framework (#133)

Signed-off-by: bitliu <bitliu@tencent.com>

test/ingress/conformance/base/manifests.yaml

@@ -0,0 +1,320 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains the base resources that most conformance tests will rely
+# on. This includes 3 namespaces along with Gateways, Services and Deployments
+# that can be used as backends for routing traffic. The most important
+# resources included are the Gateways (all in the higress-conformance-infra
+# namespace):
+# - same-namespace (only supports route in same ns)
+# - all-namespaces (supports routes in all ns)
+# - backend-namespaces (supports routes in ns with backend label)
+
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: higress-conformance-infra
+  labels:
+    higress-conformance: infra
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: infra-backend-v1
+  namespace: higress-conformance-infra
+spec:
+  selector:
+    app: infra-backend-v1
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: infra-backend-v1
+  namespace: higress-conformance-infra
+  labels:
+    app: infra-backend-v1
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: infra-backend-v1
+  template:
+    metadata:
+      labels:
+        app: infra-backend-v1
+    spec:
+      containers:
+      - name: infra-backend-v1
+        # From https://github.com/kubernetes-sigs/ingress-controller-conformance/tree/master/images/echoserver
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: infra-backend-v2
+  namespace: higress-conformance-infra
+spec:
+  selector:
+    app: infra-backend-v2
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: infra-backend-v2
+  namespace: higress-conformance-infra
+  labels:
+    app: infra-backend-v2
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: infra-backend-v2
+  template:
+    metadata:
+      labels:
+        app: infra-backend-v2
+    spec:
+      containers:
+      - name: infra-backend-v2
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: infra-backend-v3
+  namespace: higress-conformance-infra
+spec:
+  selector:
+    app: infra-backend-v3
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: infra-backend-v3
+  namespace: higress-conformance-infra
+  labels:
+    app: infra-backend-v3
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: infra-backend-v3
+  template:
+    metadata:
+      labels:
+        app: infra-backend-v3
+    spec:
+      containers:
+      - name: infra-backend-v3
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: higress-conformance-app-backend
+  labels:
+    higress-conformance: backend
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: app-backend-v1
+  namespace: higress-conformance-app-backend
+spec:
+  selector:
+    app: app-backend-v1
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app-backend-v1
+  namespace: higress-conformance-app-backend
+  labels:
+    app: app-backend-v1
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: app-backend-v1
+  template:
+    metadata:
+      labels:
+        app: app-backend-v1
+    spec:
+      containers:
+      - name: app-backend-v1
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: app-backend-v2
+  namespace: higress-conformance-app-backend
+spec:
+  selector:
+    app: app-backend-v2
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: app-backend-v2
+  namespace: higress-conformance-app-backend
+  labels:
+    app: app-backend-v2
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: app-backend-v2
+  template:
+    metadata:
+      labels:
+        app: app-backend-v2
+    spec:
+      containers:
+      - name: app-backend-v2
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: higress-conformance-web-backend
+  labels:
+    higress-conformance: backend
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web-backend
+  namespace: higress-conformance-web-backend
+spec:
+  selector:
+    app: web-backend
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web-backend
+  namespace: higress-conformance-web-backend
+  labels:
+    app: web-backend
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: web-backend
+  template:
+    metadata:
+      labels:
+        app: web-backend
+    spec:
+      containers:
+      - name: web-backend
+        image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+        env:
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+          requests:
+            cpu: 10m

test/ingress/conformance/embed.go

@@ -0,0 +1,20 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ingress
+
+import "embed"
+
+//go:embed tests/* base/*
+var Manifests embed.FS

test/ingress/conformance/tests/httproute-simple-same-namespace.go

@@ -0,0 +1,43 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import (
+	"testing"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/http"
+	"github.com/alibaba/higress/test/ingress/conformance/utils/suite"
+)
+
+func init() {
+	HigressConformanceTests = append(HigressConformanceTests, HTTPRouteSimpleSameNamespace)
+}
+
+var HTTPRouteSimpleSameNamespace = suite.ConformanceTest{
+	ShortName:   "HTTPRouteSimpleSameNamespace",
+	Description: "A single Ingress in the higress-conformance-infra namespace attaches to a Gateway in the same namespace",
+	Manifests:   []string{"tests/httproute-simple-same-namespace.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+
+		t.Run("Simple HTTP request should reach infra-backend", func(t *testing.T) {
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, http.ExpectedResponse{
+				Request:   http.Request{Path: "/hello-world"},
+				Response:  http.Response{StatusCode: 200},
+				Backend:   "infra-backend-v1",
+				Namespace: "higress-conformance-infra",
+			})
+		})
+	},
+}

test/ingress/conformance/tests/httproute-simple-same-namespace.yaml

@@ -0,0 +1,30 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: higress-conformance-infra-test
+  namespace: higress-conformance-infra
+spec:
+  rules:
+  - http:
+      paths:
+      - pathType: Prefix
+        path: "/hello-world"
+        backend:
+          service:
+            name: infra-backend-v1
+            port:
+              number: 8080

test/ingress/conformance/tests/main.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import "github.com/alibaba/higress/test/ingress/conformance/utils/suite"
+
+var HigressConformanceTests []suite.ConformanceTest

test/ingress/conformance/utils/config/timeout.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import "time"
+
+type TimeoutConfig struct {
+	// CreateTimeout represents the maximum time for a Kubernetes object to be created.
+	// Max value for conformant implementation: None
+	CreateTimeout time.Duration
+
+	// DeleteTimeout represents the maximum time for a Kubernetes object to be deleted.
+	// Max value for conformant implementation: None
+	DeleteTimeout time.Duration
+
+	// GetTimeout represents the maximum time to get a Kubernetes object.
+	// Max value for conformant implementation: None
+	GetTimeout time.Duration
+
+	// GatewayMustHaveAddress represents the maximum time for at least one IP Address has been set in the status of a Gateway.
+	// Max value for conformant implementation: None
+	GatewayMustHaveAddress time.Duration
+
+	// GatewayStatusMustHaveListeners represents the maximum time for a Gateway to have listeners in status that match the expected listeners.
+	// Max value for conformant implementation: None
+	GatewayStatusMustHaveListeners time.Duration
+
+	// GWCMustBeAccepted represents the maximum time for a GatewayClass to have an Accepted condition set to true.
+	// Max value for conformant implementation: None
+	GWCMustBeAccepted time.Duration
+
+	// HTTPRouteMustNotHaveParents represents the maximum time for an HTTPRoute to have either no parents or a single parent that is not accepted.
+	// Max value for conformant implementation: None
+	HTTPRouteMustNotHaveParents time.Duration
+
+	// HTTPRouteMustHaveCondition represents the maximum time for an HTTPRoute to have the supplied Condition.
+	// Max value for conformant implementation: None
+	HTTPRouteMustHaveCondition time.Duration
+
+	// HTTPRouteMustHaveParents represents the maximum time for an HTTPRoute to have parents in status that match the expected parents.
+	// Max value for conformant implementation: None
+	HTTPRouteMustHaveParents time.Duration
+
+	// ManifestFetchTimeout represents the maximum time for getting content from a https:// URL.
+	// Max value for conformant implementation: None
+	ManifestFetchTimeout time.Duration
+
+	// MaxTimeToConsistency is the maximum time for requiredConsecutiveSuccesses (default 3) requests to succeed in a row before failing the test.
+	// Max value for conformant implementation: 30 seconds
+	MaxTimeToConsistency time.Duration
+
+	// NamespacesMustBeReady represents the maximum time for all Pods and Gateways in a namespaces to be marked as ready.
+	// Max value for conformant implementation: None
+	NamespacesMustBeReady time.Duration
+
+	// RequestTimeout represents the maximum time for making an HTTP Request with the roundtripper.
+	// Max value for conformant implementation: None
+	RequestTimeout time.Duration
+}
+
+// DefaultTimeoutConfig populates a TimeoutConfig with the default values.
+func DefaultTimeoutConfig() TimeoutConfig {
+	return TimeoutConfig{
+		CreateTimeout:                  60 * time.Second,
+		DeleteTimeout:                  10 * time.Second,
+		GetTimeout:                     10 * time.Second,
+		GatewayMustHaveAddress:         180 * time.Second,
+		GatewayStatusMustHaveListeners: 60 * time.Second,
+		GWCMustBeAccepted:              180 * time.Second,
+		HTTPRouteMustNotHaveParents:    60 * time.Second,
+		HTTPRouteMustHaveCondition:     60 * time.Second,
+		HTTPRouteMustHaveParents:       60 * time.Second,
+		ManifestFetchTimeout:           10 * time.Second,
+		MaxTimeToConsistency:           30 * time.Second,
+		NamespacesMustBeReady:          300 * time.Second,
+		RequestTimeout:                 10 * time.Second,
+	}
+}
+
+func SetupTimeoutConfig(timeoutConfig *TimeoutConfig) {
+	defaultTimeoutConfig := DefaultTimeoutConfig()
+	if timeoutConfig.CreateTimeout == 0 {
+		timeoutConfig.CreateTimeout = defaultTimeoutConfig.CreateTimeout
+	}
+	if timeoutConfig.DeleteTimeout == 0 {
+		timeoutConfig.DeleteTimeout = defaultTimeoutConfig.DeleteTimeout
+	}
+	if timeoutConfig.GetTimeout == 0 {
+		timeoutConfig.GetTimeout = defaultTimeoutConfig.GetTimeout
+	}
+	if timeoutConfig.GatewayMustHaveAddress == 0 {
+		timeoutConfig.GatewayMustHaveAddress = defaultTimeoutConfig.GatewayMustHaveAddress
+	}
+	if timeoutConfig.GatewayStatusMustHaveListeners == 0 {
+		timeoutConfig.GatewayStatusMustHaveListeners = defaultTimeoutConfig.GatewayStatusMustHaveListeners
+	}
+	if timeoutConfig.GWCMustBeAccepted == 0 {
+		timeoutConfig.GWCMustBeAccepted = defaultTimeoutConfig.GWCMustBeAccepted
+	}
+	if timeoutConfig.HTTPRouteMustNotHaveParents == 0 {
+		timeoutConfig.HTTPRouteMustNotHaveParents = defaultTimeoutConfig.HTTPRouteMustNotHaveParents
+	}
+	if timeoutConfig.HTTPRouteMustHaveCondition == 0 {
+		timeoutConfig.HTTPRouteMustHaveCondition = defaultTimeoutConfig.HTTPRouteMustHaveCondition
+	}
+	if timeoutConfig.HTTPRouteMustHaveParents == 0 {
+		timeoutConfig.HTTPRouteMustHaveParents = defaultTimeoutConfig.HTTPRouteMustHaveParents
+	}
+	if timeoutConfig.ManifestFetchTimeout == 0 {
+		timeoutConfig.ManifestFetchTimeout = defaultTimeoutConfig.ManifestFetchTimeout
+	}
+	if timeoutConfig.MaxTimeToConsistency == 0 {
+		timeoutConfig.MaxTimeToConsistency = defaultTimeoutConfig.MaxTimeToConsistency
+	}
+	if timeoutConfig.NamespacesMustBeReady == 0 {
+		timeoutConfig.NamespacesMustBeReady = defaultTimeoutConfig.NamespacesMustBeReady
+	}
+	if timeoutConfig.RequestTimeout == 0 {
+		timeoutConfig.RequestTimeout = defaultTimeoutConfig.RequestTimeout
+	}
+}

test/ingress/conformance/utils/flags/flags.go

@@ -0,0 +1,27 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package flags
+
+import (
+	"flag"
+)
+
+var (
+	IngressClassName     = flag.String("ingress-class", "higress", "Name of IngressClass to use for tests")
+	ShowDebug            = flag.Bool("debug", false, "Whether to print debug logs")
+	CleanupBaseResources = flag.Bool("cleanup-base-resources", true, "Whether to cleanup base test resources after the run")
+	SupportedFeatures    = flag.String("supported-features", "", "Supported features included in conformance tests suites")
+	ExemptFeatures       = flag.String("exempt-features", "", "Exempt Features excluded from conformance tests suites")
+)

test/ingress/conformance/utils/http/http.go

@@ -0,0 +1,354 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/config"
+	"github.com/alibaba/higress/test/ingress/conformance/utils/roundtripper"
+)
+
+// ExpectedResponse defines the response expected for a given request.
+type ExpectedResponse struct {
+	// Request defines the request to make.
+	Request Request
+
+	// ExpectedRequest defines the request that
+	// is expected to arrive at the backend. If
+	// not specified, the backend request will be
+	// expected to match Request.
+	ExpectedRequest *ExpectedRequest
+
+	RedirectRequest *roundtripper.RedirectRequest
+
+	// BackendSetResponseHeaders is a set of headers
+	// the echoserver should set in its response.
+	BackendSetResponseHeaders map[string]string
+
+	// Response defines what response the test case
+	// should receive.
+	Response Response
+
+	Backend   string
+	Namespace string
+
+	// User Given TestCase name
+	TestCaseName string
+}
+
+// Request can be used as both the request to make and a means to verify
+// that echoserver received the expected request. Note that multiple header
+// values can be provided, as a comma-separated value.
+type Request struct {
+	Host             string
+	Method           string
+	Path             string
+	Headers          map[string]string
+	UnfollowRedirect bool
+}
+
+// ExpectedRequest defines expected properties of a request that reaches a backend.
+type ExpectedRequest struct {
+	Request
+
+	// AbsentHeaders are names of headers that are expected
+	// *not* to be present on the request.
+	AbsentHeaders []string
+}
+
+// Response defines expected properties of a response from a backend.
+type Response struct {
+	StatusCode    int
+	Headers       map[string]string
+	AbsentHeaders []string
+}
+
+// requiredConsecutiveSuccesses is the number of requests that must succeed in a row
+// for MakeRequestAndExpectEventuallyConsistentResponse to consider the response "consistent"
+// before making additional assertions on the response body. If this number is not reached within
+// maxTimeToConsistency, the test will fail.
+const requiredConsecutiveSuccesses = 3
+
+// MakeRequestAndExpectEventuallyConsistentResponse makes a request with the given parameters,
+// understanding that the request may fail for some amount of time.
+//
+// Once the request succeeds consistently with the response having the expected status code, make
+// additional assertions on the response body using the provided ExpectedResponse.
+func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripper.RoundTripper, timeoutConfig config.TimeoutConfig, gwAddr string, expected ExpectedResponse) {
+	t.Helper()
+
+	if expected.Request.Method == "" {
+		expected.Request.Method = "GET"
+	}
+
+	if expected.Response.StatusCode == 0 {
+		expected.Response.StatusCode = 200
+	}
+
+	t.Logf("Making %s request to http://%s%s", expected.Request.Method, gwAddr, expected.Request.Path)
+
+	path, query, _ := strings.Cut(expected.Request.Path, "?")
+
+	req := roundtripper.Request{
+		Method:           expected.Request.Method,
+		Host:             expected.Request.Host,
+		URL:              url.URL{Scheme: "http", Host: gwAddr, Path: path, RawQuery: query},
+		Protocol:         "HTTP",
+		Headers:          map[string][]string{},
+		UnfollowRedirect: expected.Request.UnfollowRedirect,
+	}
+
+	if expected.Request.Headers != nil {
+		for name, value := range expected.Request.Headers {
+			req.Headers[name] = []string{value}
+		}
+	}
+
+	backendSetHeaders := []string{}
+	for name, val := range expected.BackendSetResponseHeaders {
+		backendSetHeaders = append(backendSetHeaders, name+":"+val)
+	}
+	req.Headers["X-Echo-Set-Header"] = []string{strings.Join(backendSetHeaders, ",")}
+
+	WaitForConsistentResponse(t, r, req, expected, requiredConsecutiveSuccesses, timeoutConfig.MaxTimeToConsistency)
+}
+
+// awaitConvergence runs the given function until it returns 'true' `threshold` times in a row.
+// Each failed attempt has a 1s delay; successful attempts have no delay.
+func awaitConvergence(t *testing.T, threshold int, maxTimeToConsistency time.Duration, fn func(elapsed time.Duration) bool) {
+	successes := 0
+	attempts := 0
+	start := time.Now()
+	to := time.After(maxTimeToConsistency)
+	delay := time.Second
+	for {
+		select {
+		case <-to:
+			t.Fatalf("timeout while waiting after %d attempts", attempts)
+		default:
+		}
+
+		completed := fn(time.Now().Sub(start))
+		attempts++
+		if completed {
+			successes++
+			if successes >= threshold {
+				return
+			}
+			// Skip delay if we have a success
+			continue
+		}
+
+		successes = 0
+		select {
+		// Capture the overall timeout
+		case <-to:
+			t.Fatalf("timeout while waiting after %d attempts, %d/%d sucessess", attempts, successes, threshold)
+			// And the per-try delay
+		case <-time.After(delay):
+		}
+	}
+}
+
+// WaitForConsistentResponse repeats the provided request until it completes with a response having
+// the expected response consistently. The provided threshold determines how many times in
+// a row this must occur to be considered "consistent".
+func WaitForConsistentResponse(t *testing.T, r roundtripper.RoundTripper, req roundtripper.Request, expected ExpectedResponse, threshold int, maxTimeToConsistency time.Duration) {
+	awaitConvergence(t, threshold, maxTimeToConsistency, func(elapsed time.Duration) bool {
+		cReq, cRes, err := r.CaptureRoundTrip(req)
+		if err != nil {
+			t.Logf("Request failed, not ready yet: %v (after %v)", err.Error(), elapsed)
+			return false
+		}
+
+		if err := CompareRequest(&req, cReq, cRes, expected); err != nil {
+			t.Logf("Response expectation failed for request: %v  not ready yet: %v (after %v)", req, err, elapsed)
+			return false
+		}
+
+		return true
+	})
+	t.Logf("Request passed")
+}
+
+func CompareRequest(req *roundtripper.Request, cReq *roundtripper.CapturedRequest, cRes *roundtripper.CapturedResponse, expected ExpectedResponse) error {
+	if expected.Response.StatusCode != cRes.StatusCode {
+		return fmt.Errorf("expected status code to be %d, got %d", expected.Response.StatusCode, cRes.StatusCode)
+	}
+	if cRes.StatusCode == 200 {
+		// The request expected to arrive at the backend is
+		// the same as the request made, unless otherwise
+		// specified.
+		if expected.ExpectedRequest == nil {
+			expected.ExpectedRequest = &ExpectedRequest{Request: expected.Request}
+		}
+
+		if expected.ExpectedRequest.Method == "" {
+			expected.ExpectedRequest.Method = "GET"
+		}
+
+		if expected.ExpectedRequest.Host != "" && expected.ExpectedRequest.Host != cReq.Host {
+			return fmt.Errorf("expected host to be %s, got %s", expected.ExpectedRequest.Host, cReq.Host)
+		}
+
+		if expected.ExpectedRequest.Path != cReq.Path {
+			return fmt.Errorf("expected path to be %s, got %s", expected.ExpectedRequest.Path, cReq.Path)
+		}
+		if expected.ExpectedRequest.Method != cReq.Method {
+			return fmt.Errorf("expected method to be %s, got %s", expected.ExpectedRequest.Method, cReq.Method)
+		}
+		if expected.Namespace != cReq.Namespace {
+			return fmt.Errorf("expected namespace to be %s, got %s", expected.Namespace, cReq.Namespace)
+		}
+		if expected.ExpectedRequest.Headers != nil {
+			if cReq.Headers == nil {
+				return fmt.Errorf("no headers captured, expected %v", len(expected.ExpectedRequest.Headers))
+			}
+			for name, val := range cReq.Headers {
+				cReq.Headers[strings.ToLower(name)] = val
+			}
+			for name, expectedVal := range expected.ExpectedRequest.Headers {
+				actualVal, ok := cReq.Headers[strings.ToLower(name)]
+				if !ok {
+					return fmt.Errorf("expected %s header to be set, actual headers: %v", name, cReq.Headers)
+				} else if strings.Join(actualVal, ",") != expectedVal {
+					return fmt.Errorf("expected %s header to be set to %s, got %s", name, expectedVal, strings.Join(actualVal, ","))
+				}
+
+			}
+		}
+
+		if expected.Response.Headers != nil {
+			if cRes.Headers == nil {
+				return fmt.Errorf("no headers captured, expected %v", len(expected.ExpectedRequest.Headers))
+			}
+			for name, val := range cRes.Headers {
+				cRes.Headers[strings.ToLower(name)] = val
+			}
+
+			for name, expectedVal := range expected.Response.Headers {
+				actualVal, ok := cRes.Headers[strings.ToLower(name)]
+				if !ok {
+					return fmt.Errorf("expected %s header to be set, actual headers: %v", name, cRes.Headers)
+				} else if strings.Join(actualVal, ",") != expectedVal {
+					return fmt.Errorf("expected %s header to be set to %s, got %s", name, expectedVal, strings.Join(actualVal, ","))
+				}
+			}
+		}
+
+		if len(expected.Response.AbsentHeaders) > 0 {
+			for name, val := range cRes.Headers {
+				cRes.Headers[strings.ToLower(name)] = val
+			}
+
+			for _, name := range expected.Response.AbsentHeaders {
+				val, ok := cRes.Headers[strings.ToLower(name)]
+				if ok {
+					return fmt.Errorf("expected %s header to not be set, got %s", name, val)
+				}
+			}
+		}
+
+		// Verify that headers expected *not* to be present on the
+		// request are actually not present.
+		if len(expected.ExpectedRequest.AbsentHeaders) > 0 {
+			for name, val := range cReq.Headers {
+				cReq.Headers[strings.ToLower(name)] = val
+			}
+
+			for _, name := range expected.ExpectedRequest.AbsentHeaders {
+				val, ok := cReq.Headers[strings.ToLower(name)]
+				if ok {
+					return fmt.Errorf("expected %s header to not be set, got %s", name, val)
+				}
+			}
+		}
+
+		if !strings.HasPrefix(cReq.Pod, expected.Backend) {
+			return fmt.Errorf("expected pod name to start with %s, got %s", expected.Backend, cReq.Pod)
+		}
+	} else if roundtripper.IsRedirect(cRes.StatusCode) {
+		if expected.RedirectRequest == nil {
+			return nil
+		}
+
+		setRedirectRequestDefaults(req, cRes, &expected)
+
+		if expected.RedirectRequest.Host != cRes.RedirectRequest.Host {
+			return fmt.Errorf("expected redirected hostname to be %s, got %s", expected.RedirectRequest.Host, cRes.RedirectRequest.Host)
+		}
+
+		if expected.RedirectRequest.Port != cRes.RedirectRequest.Port {
+			return fmt.Errorf("expected redirected port to be %s, got %s", expected.RedirectRequest.Port, cRes.RedirectRequest.Port)
+		}
+
+		if expected.RedirectRequest.Scheme != cRes.RedirectRequest.Scheme {
+			return fmt.Errorf("expected redirected scheme to be %s, got %s", expected.RedirectRequest.Scheme, cRes.RedirectRequest.Scheme)
+		}
+
+		if expected.RedirectRequest.Path != cRes.RedirectRequest.Path {
+			return fmt.Errorf("expected redirected path to be %s, got %s", expected.RedirectRequest.Path, cRes.RedirectRequest.Path)
+		}
+	}
+	return nil
+}
+
+// Get User-defined test case name or generate from expected response to a given request.
+func (er *ExpectedResponse) GetTestCaseName(i int) string {
+
+	// If TestCase name is provided then use that or else generate one.
+	if er.TestCaseName != "" {
+		return er.TestCaseName
+	}
+
+	headerStr := ""
+	reqStr := ""
+
+	if er.Request.Headers != nil {
+		headerStr = " with headers"
+	}
+
+	reqStr = fmt.Sprintf("%d request to '%s%s'%s", i, er.Request.Host, er.Request.Path, headerStr)
+
+	if er.Backend != "" {
+		return fmt.Sprintf("%s should go to %s", reqStr, er.Backend)
+	}
+	return fmt.Sprintf("%s should receive a %d", reqStr, er.Response.StatusCode)
+}
+
+func setRedirectRequestDefaults(req *roundtripper.Request, cRes *roundtripper.CapturedResponse, expected *ExpectedResponse) {
+	// If the expected host is nil it means we do not test host redirect.
+	// In that case we are setting it to the one we got from the response because we do not know the ip/host of the gateway.
+	if expected.RedirectRequest.Host == "" {
+		expected.RedirectRequest.Host = cRes.RedirectRequest.Host
+	}
+
+	if expected.RedirectRequest.Port == "" {
+		expected.RedirectRequest.Port = req.URL.Port()
+	}
+
+	if expected.RedirectRequest.Scheme == "" {
+		expected.RedirectRequest.Scheme = req.URL.Scheme
+	}
+
+	if expected.RedirectRequest.Path == "" {
+		expected.RedirectRequest.Path = req.URL.Path
+	}
+}

test/ingress/conformance/utils/kubernetes/apply.go

@@ -0,0 +1,229 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	ingress "github.com/alibaba/higress/test/ingress/conformance"
+	"github.com/stretchr/testify/require"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/yaml"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/config"
+)
+
+// Applier prepares manifests depending on the available options and applies
+// them to the Kubernetes cluster.
+type Applier struct {
+	NamespaceLabels map[string]string
+	// ValidUniqueListenerPorts maps each listener port of each Gateway in the
+	// manifests to a valid, unique port. There must be as many
+	// ValidUniqueListenerPorts as there are listeners in the set of manifests.
+	// For example, given two Gateways, each with 2 listeners, there should be
+	// four ValidUniqueListenerPorts.
+	// If empty or nil, ports are not modified.
+	ValidUniqueListenerPorts []int
+
+	// IngressClass will be used as the spec.gatewayClassName when applying Gateway resources
+	IngressClass string
+
+	// ControllerName will be used as the spec.controllerName when applying GatewayClass resources
+	ControllerName string
+}
+
+// prepareNamespace adjusts the Namespace labels.
+func prepareNamespace(t *testing.T, uObj *unstructured.Unstructured, namespaceLabels map[string]string) {
+	labels, _, err := unstructured.NestedStringMap(uObj.Object, "metadata", "labels")
+	require.NoErrorf(t, err, "error getting labels on Namespace %s", uObj.GetName())
+
+	for k, v := range namespaceLabels {
+		if labels == nil {
+			labels = map[string]string{}
+		}
+
+		labels[k] = v
+	}
+
+	// SetNestedStringMap converts nil to an empty map
+	if labels != nil {
+		err = unstructured.SetNestedStringMap(uObj.Object, labels, "metadata", "labels")
+	}
+	require.NoErrorf(t, err, "error setting labels on Namespace %s", uObj.GetName())
+}
+
+// prepareResources uses the options from an Applier to tweak resources given by
+// a set of manifests.
+func (a Applier) prepareResources(t *testing.T, decoder *yaml.YAMLOrJSONDecoder) ([]unstructured.Unstructured, error) {
+	var resources []unstructured.Unstructured
+
+	for {
+		uObj := unstructured.Unstructured{}
+		if err := decoder.Decode(&uObj); err != nil {
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			return nil, err
+		}
+		if len(uObj.Object) == 0 {
+			continue
+		}
+
+		if uObj.GetKind() == "Namespace" && uObj.GetObjectKind().GroupVersionKind().Group == "" {
+			prepareNamespace(t, &uObj, a.NamespaceLabels)
+		}
+
+		resources = append(resources, uObj)
+	}
+
+	return resources, nil
+}
+
+func (a Applier) MustApplyObjectsWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, resources []client.Object, cleanup bool) {
+	for _, resource := range resources {
+		resource := resource
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout)
+		defer cancel()
+
+		t.Logf("Creating %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind)
+
+		err := c.Create(ctx, resource)
+		if err != nil {
+			if !apierrors.IsAlreadyExists(err) {
+				require.NoError(t, err, "error creating resource")
+			}
+		}
+
+		if cleanup {
+			t.Cleanup(func() {
+				ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+				defer cancel()
+				t.Logf("Deleting %s %s", resource.GetName(), resource.GetObjectKind().GroupVersionKind().Kind)
+				err = c.Delete(ctx, resource)
+				require.NoErrorf(t, err, "error deleting resource")
+			})
+		}
+	}
+}
+
+// MustApplyWithCleanup creates or updates Kubernetes resources defined with the
+// provided YAML file and registers a cleanup function for resources it created.
+// Note that this does not remove resources that already existed in the cluster.
+func (a Applier) MustApplyWithCleanup(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, location string, cleanup bool) {
+	data, err := getContentsFromPathOrURL(location, timeoutConfig)
+	require.NoError(t, err)
+
+	decoder := yaml.NewYAMLOrJSONDecoder(data, 4096)
+
+	resources, err := a.prepareResources(t, decoder)
+	if err != nil {
+		t.Logf("manifest: %s", data.String())
+		require.NoErrorf(t, err, "error parsing manifest")
+	}
+
+	for i := range resources {
+		uObj := &resources[i]
+
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.CreateTimeout)
+		defer cancel()
+
+		namespacedName := types.NamespacedName{Namespace: uObj.GetNamespace(), Name: uObj.GetName()}
+		fetchedObj := uObj.DeepCopy()
+		err := c.Get(ctx, namespacedName, fetchedObj)
+		if err != nil {
+			if !apierrors.IsNotFound(err) {
+				require.NoErrorf(t, err, "error getting resource")
+			}
+			t.Logf("Creating %s %s", uObj.GetName(), uObj.GetKind())
+			err = c.Create(ctx, uObj)
+			require.NoErrorf(t, err, "error creating resource")
+
+			if cleanup {
+				t.Cleanup(func() {
+					ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+					defer cancel()
+					t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind())
+					err = c.Delete(ctx, uObj)
+					require.NoErrorf(t, err, "error deleting resource")
+				})
+			}
+			continue
+		}
+
+		uObj.SetResourceVersion(fetchedObj.GetResourceVersion())
+		t.Logf("Updating %s %s", uObj.GetName(), uObj.GetKind())
+		err = c.Update(ctx, uObj)
+
+		if cleanup {
+			t.Cleanup(func() {
+				ctx, cancel = context.WithTimeout(context.Background(), timeoutConfig.DeleteTimeout)
+				defer cancel()
+				t.Logf("Deleting %s %s", uObj.GetName(), uObj.GetKind())
+				err = c.Delete(ctx, uObj)
+				require.NoErrorf(t, err, "error deleting resource")
+			})
+		}
+		require.NoErrorf(t, err, "error updating resource")
+	}
+}
+
+// getContentsFromPathOrURL takes a string that can either be a local file
+// path or an https:// URL to YAML manifests and provides the contents.
+func getContentsFromPathOrURL(location string, timeoutConfig config.TimeoutConfig) (*bytes.Buffer, error) {
+	if strings.HasPrefix(location, "http://") {
+		return nil, fmt.Errorf("data can't be retrieved from %s: http is not supported, use https", location)
+	} else if strings.HasPrefix(location, "https://") {
+		ctx, cancel := context.WithTimeout(context.Background(), timeoutConfig.ManifestFetchTimeout)
+		defer cancel()
+
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			return nil, err
+		}
+		defer resp.Body.Close()
+
+		manifests := new(bytes.Buffer)
+		count, err := manifests.ReadFrom(resp.Body)
+		if err != nil {
+			return nil, err
+		}
+
+		if resp.ContentLength != -1 && count != resp.ContentLength {
+			return nil, fmt.Errorf("received %d bytes from %s, expected %d", count, location, resp.ContentLength)
+		}
+		return manifests, nil
+	}
+	b, err := ingress.Manifests.ReadFile(location)
+	if err != nil {
+		return nil, err
+	}
+	return bytes.NewBuffer(b), nil
+}

test/ingress/conformance/utils/kubernetes/apply_test.go

@@ -0,0 +1,19 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	_ "github.com/alibaba/higress/test/ingress/conformance/utils/flags"
+)

test/ingress/conformance/utils/kubernetes/cert.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"bytes"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	// ensure auth plugins are loaded
+	_ "k8s.io/client-go/plugin/pkg/client/auth"
+)
+
+const (
+	rsaBits  = 2048
+	validFor = 365 * 24 * time.Hour
+)
+
+// MustCreateSelfSignedCertSecret creates a self-signed SSL certificate and stores it in a secret
+func MustCreateSelfSignedCertSecret(t *testing.T, namespace, secretName string, hosts []string) *corev1.Secret {
+	require.Greater(t, len(hosts), 0, "require a non-empty hosts for Subject Alternate Name values")
+
+	var serverKey, serverCert bytes.Buffer
+
+	host := strings.Join(hosts, ",")
+
+	require.NoError(t, generateRSACert(host, &serverKey, &serverCert), "failed to generate RSA certificate")
+
+	data := map[string][]byte{
+		corev1.TLSCertKey:       serverCert.Bytes(),
+		corev1.TLSPrivateKeyKey: serverKey.Bytes(),
+	}
+
+	newSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: namespace,
+			Name:      secretName,
+		},
+		Type: corev1.SecretTypeTLS,
+		Data: data,
+	}
+
+	return newSecret
+}
+
+// generateRSACert generates a basic self signed certificate valir for a year
+func generateRSACert(host string, keyOut, certOut io.Writer) error {
+	priv, err := rsa.GenerateKey(rand.Reader, rsaBits)
+	if err != nil {
+		return fmt.Errorf("failed to generate key: %w", err)
+	}
+	notBefore := time.Now()
+	notAfter := notBefore.Add(validFor)
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
+
+	if err != nil {
+		return fmt.Errorf("failed to generate serial number: %w", err)
+	}
+
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			CommonName:   "default",
+			Organization: []string{"Acme Co"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+
+	if err != nil {
+		return fmt.Errorf("failed to create certificate: %w", err)
+	}
+
+	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		return fmt.Errorf("failed creating cert: %w", err)
+	}
+
+	if err := pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)}); err != nil {
+		return fmt.Errorf("failed creating key: %w", err)
+	}
+
+	return nil
+}

test/ingress/conformance/utils/kubernetes/helpers.go

@@ -0,0 +1,122 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kubernetes
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/config"
+)
+
+// FilterStaleConditions returns the list of status condition whos observedGeneration does not
+// match the objects metadata.Generation
+func FilterStaleConditions(obj metav1.Object, conditions []metav1.Condition) []metav1.Condition {
+	stale := make([]metav1.Condition, 0, len(conditions))
+	for _, condition := range conditions {
+		if obj.GetGeneration() != condition.ObservedGeneration {
+			stale = append(stale, condition)
+		}
+	}
+	return stale
+}
+
+// NamespacesMustBeAccepted waits until all Pods are marked ready
+// in the provided namespaces. This will cause the test to
+// halt if the specified timeout is exceeded.
+func NamespacesMustBeAccepted(t *testing.T, c client.Client, timeoutConfig config.TimeoutConfig, namespaces []string) {
+	t.Helper()
+
+	waitErr := wait.PollImmediate(1*time.Second, timeoutConfig.NamespacesMustBeReady, func() (bool, error) {
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		for _, ns := range namespaces {
+			podList := &v1.PodList{}
+			err := c.List(ctx, podList, client.InNamespace(ns))
+			if err != nil {
+				t.Errorf("Error listing Pods: %v", err)
+			}
+			for _, pod := range podList.Items {
+				if !FindPodConditionInList(t, pod.Status.Conditions, "Ready", "True") &&
+					pod.Status.Phase != v1.PodSucceeded {
+					t.Logf("%s/%s Pod not ready yet", ns, pod.Name)
+					return false, nil
+				}
+			}
+		}
+		t.Logf("Gateways and Pods in %s namespaces ready", strings.Join(namespaces, ", "))
+		return true, nil
+	})
+	require.NoErrorf(t, waitErr, "error waiting for %s namespaces to be ready", strings.Join(namespaces, ", "))
+}
+
+func ConditionsMatch(t *testing.T, expected, actual []metav1.Condition) bool {
+	if len(actual) < len(expected) {
+		t.Logf("Expected more conditions to be present")
+		return false
+	}
+	for _, condition := range expected {
+		if !FindConditionInList(t, actual, condition.Type, string(condition.Status), condition.Reason) {
+			return false
+		}
+	}
+
+	t.Logf("Conditions matched expectations")
+	return true
+}
+
+// findConditionInList finds a condition in a list of Conditions, checking
+// the Name, Value, and Reason. If an empty reason is passed, any Reason will match.
+func FindConditionInList(t *testing.T, conditions []metav1.Condition, condName, expectedStatus, expectedReason string) bool {
+	for _, cond := range conditions {
+		if cond.Type == condName {
+			if cond.Status == metav1.ConditionStatus(expectedStatus) {
+				// an empty Reason string means "Match any reason".
+				if expectedReason == "" || cond.Reason == expectedReason {
+					return true
+				}
+				t.Logf("%s condition Reason set to %s, expected %s", condName, cond.Reason, expectedReason)
+			}
+
+			t.Logf("%s condition set to Status %s with Reason %v, expected Status %s", condName, cond.Status, cond.Reason, expectedStatus)
+		}
+	}
+
+	t.Logf("%s was not in conditions list", condName)
+	return false
+}
+
+func FindPodConditionInList(t *testing.T, conditions []v1.PodCondition, condName, condValue string) bool {
+	for _, cond := range conditions {
+		if cond.Type == v1.PodConditionType(condName) {
+			if cond.Status == v1.ConditionStatus(condValue) {
+				return true
+			}
+			t.Logf("%s condition set to %s, expected %s", condName, cond.Status, condValue)
+		}
+	}
+
+	t.Logf("%s was not in conditions list", condName)
+	return false
+}

test/ingress/conformance/utils/roundtripper/roundtripper.go

@@ -0,0 +1,199 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package roundtripper
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"regexp"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/config"
+)
+
+// RoundTripper is an interface used to make requests within conformance tests.
+// This can be overridden with custom implementations whenever necessary.
+type RoundTripper interface {
+	CaptureRoundTrip(Request) (*CapturedRequest, *CapturedResponse, error)
+}
+
+// Request is the primary input for making a request.
+type Request struct {
+	URL              url.URL
+	Host             string
+	Protocol         string
+	Method           string
+	Headers          map[string][]string
+	UnfollowRedirect bool
+}
+
+// CapturedRequest contains request metadata captured from an echoserver
+// response.
+type CapturedRequest struct {
+	Path     string              `json:"path"`
+	Host     string              `json:"host"`
+	Method   string              `json:"method"`
+	Protocol string              `json:"proto"`
+	Headers  map[string][]string `json:"headers"`
+
+	Namespace string `json:"namespace"`
+	Pod       string `json:"pod"`
+}
+
+// RedirectRequest contains a follow up request metadata captured from a redirect
+// response.
+type RedirectRequest struct {
+	Scheme string
+	Host   string
+	Port   string
+	Path   string
+}
+
+// CapturedResponse contains response metadata.
+type CapturedResponse struct {
+	StatusCode      int
+	ContentLength   int64
+	Protocol        string
+	Headers         map[string][]string
+	RedirectRequest *RedirectRequest
+}
+
+// DefaultRoundTripper is the default implementation of a RoundTripper. It will
+// be used if a custom implementation is not specified.
+type DefaultRoundTripper struct {
+	Debug         bool
+	TimeoutConfig config.TimeoutConfig
+}
+
+// CaptureRoundTrip makes a request with the provided parameters and returns the
+// captured request and response from echoserver. An error will be returned if
+// there is an error running the function but not if an HTTP error status code
+// is received.
+func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) {
+	cReq := &CapturedRequest{}
+	client := &http.Client{}
+
+	if request.UnfollowRedirect {
+		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		}
+	}
+
+	method := "GET"
+	if request.Method != "" {
+		method = request.Method
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), d.TimeoutConfig.RequestTimeout)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, method, request.URL.String(), nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if request.Host != "" {
+		req.Host = request.Host
+	}
+
+	if request.Headers != nil {
+		for name, value := range request.Headers {
+			req.Header.Set(name, value[0])
+		}
+	}
+
+	if d.Debug {
+		var dump []byte
+		dump, err = httputil.DumpRequestOut(req, true)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		fmt.Printf("Sending Request:\n%s\n\n", formatDump(dump, "< "))
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer resp.Body.Close()
+
+	if d.Debug {
+		var dump []byte
+		dump, err = httputil.DumpResponse(resp, true)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		fmt.Printf("Received Response:\n%s\n\n", formatDump(dump, "< "))
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+
+	// we cannot assume the response is JSON
+	if resp.Header.Get("Content-type") == "application/json" {
+		err = json.Unmarshal(body, cReq)
+		if err != nil {
+			return nil, nil, fmt.Errorf("unexpected error reading response: %w", err)
+		}
+	}
+
+	cRes := &CapturedResponse{
+		StatusCode:    resp.StatusCode,
+		ContentLength: resp.ContentLength,
+		Protocol:      resp.Proto,
+		Headers:       resp.Header,
+	}
+
+	if IsRedirect(resp.StatusCode) {
+		redirectURL, err := resp.Location()
+		if err != nil {
+			return nil, nil, err
+		}
+		cRes.RedirectRequest = &RedirectRequest{
+			Scheme: redirectURL.Scheme,
+			Host:   redirectURL.Hostname(),
+			Port:   redirectURL.Port(),
+			Path:   redirectURL.Path,
+		}
+	}
+
+	return cReq, cRes, nil
+}
+
+// IsRedirect returns true if a given status code is a redirect code.
+func IsRedirect(statusCode int) bool {
+	switch statusCode {
+	case http.StatusMultipleChoices,
+		http.StatusMovedPermanently,
+		http.StatusFound,
+		http.StatusSeeOther,
+		http.StatusNotModified,
+		http.StatusUseProxy,
+		http.StatusTemporaryRedirect,
+		http.StatusPermanentRedirect:
+		return true
+	}
+	return false
+}
+
+var startLineRegex = regexp.MustCompile(`(?m)^`)
+
+func formatDump(data []byte, prefix string) string {
+	data = startLineRegex.ReplaceAllLiteral(data, []byte(prefix))
+	return string(data)
+}

test/ingress/conformance/utils/suite/suite.go

@@ -0,0 +1,217 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package suite
+
+import (
+	"testing"
+
+	"github.com/alibaba/higress/test/ingress/conformance/utils/config"
+	"github.com/alibaba/higress/test/ingress/conformance/utils/kubernetes"
+	"github.com/alibaba/higress/test/ingress/conformance/utils/roundtripper"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+// SupportedFeature allows opting in to additional conformance tests at an
+// individual feature granularity.
+type SupportedFeature string
+
+const (
+	// This option indicates support for TLSRoute (extended conformance).
+	SupportTLSRoute SupportedFeature = "TLSRoute"
+
+	// This option indicates support for HTTPRoute query param matching (extended conformance).
+	SupportHTTPRouteQueryParamMatching SupportedFeature = "HTTPRouteQueryParamMatching"
+
+	// This option indicates support for HTTPRoute method matching (extended conformance).
+	SupportHTTPRouteMethodMatching SupportedFeature = "HTTPRouteMethodMatching"
+
+	// This option indicates support for HTTPRoute response header modification (extended conformance).
+	SupportHTTPResponseHeaderModification SupportedFeature = "HTTPResponseHeaderModification"
+
+	// This option indicates support for Destination Port matching (extended conformance).
+	SupportRouteDestinationPortMatching SupportedFeature = "RouteDestinationPortMatching"
+
+	// This option indicates support for HTTPRoute port redirect (extended conformance).
+	SupportHTTPRoutePortRedirect SupportedFeature = "HTTPRoutePortRedirect"
+
+	// This option indicates support for HTTPRoute scheme redirect (extended conformance).
+	SupportHTTPRouteSchemeRedirect SupportedFeature = "HTTPRouteSchemeRedirect"
+
+	// This option indicates support for HTTPRoute path redirect (experimental conformance).
+	SupportHTTPRoutePathRedirect SupportedFeature = "HTTPRoutePathRedirect"
+
+	// This option indicates support for HTTPRoute host rewrite (experimental conformance)
+	SupportHTTPRouteHostRewrite SupportedFeature = "HTTPRouteHostRewrite"
+
+	// This option indicates support for HTTPRoute path rewrite (experimental conformance)
+	SupportHTTPRoutePathRewrite SupportedFeature = "HTTPRoutePathRewrite"
+)
+
+// StandardCoreFeatures are the features that are required to be conformant with
+// the Core API features that are part of the Standard release channel.
+var StandardCoreFeatures = map[SupportedFeature]bool{}
+
+// ConformanceTestSuite defines the test suite used to run Gateway API
+// conformance tests.
+type ConformanceTestSuite struct {
+	Client            client.Client
+	RoundTripper      roundtripper.RoundTripper
+	GatewayAddress    string
+	IngressClassName  string
+	ControllerName    string
+	Debug             bool
+	Cleanup           bool
+	BaseManifests     string
+	Applier           kubernetes.Applier
+	SupportedFeatures map[SupportedFeature]bool
+	TimeoutConfig     config.TimeoutConfig
+}
+
+// Options can be used to initialize a ConformanceTestSuite.
+type Options struct {
+	Client           client.Client
+	GatewayAddress   string
+	IngressClassName string
+	Debug            bool
+	RoundTripper     roundtripper.RoundTripper
+	BaseManifests    string
+	NamespaceLabels  map[string]string
+	// ValidUniqueListenerPorts maps each listener port of each Gateway in the
+	// manifests to a valid, unique port. There must be as many
+	// ValidUniqueListenerPorts as there are listeners in the set of manifests.
+	// For example, given two Gateways, each with 2 listeners, there should be
+	// four ValidUniqueListenerPorts.
+	// If empty or nil, ports are not modified.
+	ValidUniqueListenerPorts []int
+
+	// CleanupBaseResources indicates whether or not the base test
+	// resources such as Gateways should be cleaned up after the run.
+	CleanupBaseResources bool
+	SupportedFeatures    map[SupportedFeature]bool
+	TimeoutConfig        config.TimeoutConfig
+}
+
+// New returns a new ConformanceTestSuite.
+func New(s Options) *ConformanceTestSuite {
+	config.SetupTimeoutConfig(&s.TimeoutConfig)
+
+	roundTripper := s.RoundTripper
+	if roundTripper == nil {
+		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
+	}
+
+	if s.SupportedFeatures == nil {
+		s.SupportedFeatures = StandardCoreFeatures
+	} else {
+		for feature, val := range StandardCoreFeatures {
+			if _, ok := s.SupportedFeatures[feature]; !ok {
+				s.SupportedFeatures[feature] = val
+			}
+		}
+	}
+
+	suite := &ConformanceTestSuite{
+		Client:           s.Client,
+		RoundTripper:     roundTripper,
+		IngressClassName: s.IngressClassName,
+		Debug:            s.Debug,
+		Cleanup:          s.CleanupBaseResources,
+		BaseManifests:    s.BaseManifests,
+		GatewayAddress:   s.GatewayAddress,
+		Applier: kubernetes.Applier{
+			NamespaceLabels:          s.NamespaceLabels,
+			ValidUniqueListenerPorts: s.ValidUniqueListenerPorts,
+		},
+		SupportedFeatures: s.SupportedFeatures,
+		TimeoutConfig:     s.TimeoutConfig,
+	}
+
+	// apply defaults
+	if suite.BaseManifests == "" {
+		suite.BaseManifests = "base/manifests.yaml"
+	}
+
+	return suite
+}
+
+// Setup ensures the base resources required for conformance tests are installed
+// in the cluster. It also ensures that all relevant resources are ready.
+func (suite *ConformanceTestSuite) Setup(t *testing.T) {
+	t.Logf("Test Setup: Ensuring IngressClass has been accepted")
+	suite.ControllerName = suite.IngressClassName
+
+	suite.Applier.IngressClass = suite.IngressClassName
+	suite.Applier.ControllerName = suite.ControllerName
+
+	t.Logf("Test Setup: Applying base manifests")
+	suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup)
+
+	t.Logf("Test Setup: Applying programmatic resources")
+	secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"})
+	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
+	secret = kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-infra", "tls-validity-checks-certificate", []string{"*"})
+	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
+
+	t.Logf("Test Setup: Ensuring Pods from base manifests are ready")
+	namespaces := []string{
+		"higress-conformance-infra",
+		"higress-conformance-app-backend",
+		"higress-conformance-web-backend",
+	}
+	kubernetes.NamespacesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, namespaces)
+}
+
+// Run runs the provided set of conformance tests.
+func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
+	for _, test := range tests {
+		t.Run(test.ShortName, func(t *testing.T) {
+			test.Run(t, suite)
+		})
+	}
+}
+
+// ConformanceTest is used to define each individual conformance test.
+type ConformanceTest struct {
+	ShortName   string
+	Description string
+	Features    []SupportedFeature
+	Manifests   []string
+	Slow        bool
+	Parallel    bool
+	Test        func(*testing.T, *ConformanceTestSuite)
+}
+
+// Run runs an individual tests, applying and cleaning up the required manifests
+// before calling the Test function.
+func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
+	if test.Parallel {
+		t.Parallel()
+	}
+
+	// Check that all features exercised by the test have been opted into by
+	// the suite.
+	for _, feature := range test.Features {
+		if supported, ok := suite.SupportedFeatures[feature]; !ok || !supported {
+			t.Skipf("Skipping %s: suite does not support %s", test.ShortName, feature)
+		}
+	}
+
+	for _, manifestLocation := range test.Manifests {
+		t.Logf("Applying %s", manifestLocation)
+		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true)
+	}
+
+	test.Test(t, suite)
+}