feat: add features to conformance and do some refactors (#532)

Signed-off-by: bitliu <bitliu@tencent.com>

test/e2e/conformance/utils/suite/features.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package suite
+
+import "istio.io/istio/pilot/pkg/util/sets"
+
+type SupportedFeature string
+
+const (
+	// core: http
+	HTTPConformanceFeature SupportedFeature = "http"
+
+	// extended: extensibility
+	WASMGoConformanceFeature  SupportedFeature = "wasm-go"
+	WASMCPPConformanceFeature SupportedFeature = "wasm-cpp"
+
+	// extended: service discovery
+	DubboConformanceFeature  SupportedFeature = "dubbo"
+	EurekaConformanceFeature SupportedFeature = "eureka"
+	ConsulConformanceFeature SupportedFeature = "consul"
+	NacosConformanceFeature  SupportedFeature = "nacos"
+)
+
+var AllFeatures = sets.Set{}.
+	Insert(string(HTTPConformanceFeature)).
+	Insert(string(DubboConformanceFeature)).
+	Insert(string(EurekaConformanceFeature)).
+	Insert(string(ConsulConformanceFeature)).
+	Insert(string(NacosConformanceFeature))
+
+var ExperimentFeatures = sets.Set{}.
+	Insert(string(WASMGoConformanceFeature)).
+	Insert(string(WASMCPPConformanceFeature))

test/e2e/conformance/utils/suite/suite.go

@@ -20,32 +20,41 @@ import (
 	"github.com/alibaba/higress/test/e2e/conformance/utils/config"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes"
 	"github.com/alibaba/higress/test/e2e/conformance/utils/roundtripper"
+	"istio.io/istio/pilot/pkg/util/sets"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // ConformanceTestSuite defines the test suite used to run Gateway API
 // conformance tests.
 type ConformanceTestSuite struct {
-	Client           client.Client
-	RoundTripper     roundtripper.RoundTripper
-	GatewayAddress   string
-	IngressClassName string
-	Debug            bool
-	Cleanup          bool
-	BaseManifests    string
-	Applier          kubernetes.Applier
-	TimeoutConfig    config.TimeoutConfig
+	Client            client.Client
+	RoundTripper      roundtripper.RoundTripper
+	GatewayAddress    string
+	IngressClassName  string
+	Debug             bool
+	Cleanup           bool
+	BaseManifests     string
+	Applier           kubernetes.Applier
+	SkipTests         sets.Set
+	TimeoutConfig     config.TimeoutConfig
+	SupportedFeatures sets.Set
 }
 
 // Options can be used to initialize a ConformanceTestSuite.
 type Options struct {
-	Client           client.Client
-	GatewayAddress   string
-	IngressClassName string
-	Debug            bool
-	RoundTripper     roundtripper.RoundTripper
-	BaseManifests    string
-	NamespaceLabels  map[string]string
+	SupportedFeatures sets.Set
+	ExemptFeatures    sets.Set
+
+	EnableAllSupportedFeatures bool
+	Client                     client.Client
+	GatewayAddress             string
+	IngressClassName           string
+	Debug                      bool
+	RoundTripper               roundtripper.RoundTripper
+	BaseManifests              string
+	NamespaceLabels            map[string]string
+	// Options for wasm extended features
+	WASMOptions
 
 	// CleanupBaseResources indicates whether or not the base test
 	// resources such as Gateways should be cleaned up after the run.
@@ -53,6 +62,12 @@ type Options struct {
 	TimeoutConfig        config.TimeoutConfig
 }
 
+type WASMOptions struct {
+	IsWasmPluginTest bool
+	WasmPluginType   string
+	WasmPluginName   string
+}
+
 // New returns a new ConformanceTestSuite.
 func New(s Options) *ConformanceTestSuite {
 	config.SetupTimeoutConfig(&s.TimeoutConfig)
@@ -62,14 +77,33 @@ func New(s Options) *ConformanceTestSuite {
 		roundTripper = &roundtripper.DefaultRoundTripper{Debug: s.Debug, TimeoutConfig: s.TimeoutConfig}
 	}
 
+	if s.SupportedFeatures == nil {
+		s.SupportedFeatures = sets.Set{}
+	}
+
+	if s.IsWasmPluginTest {
+		if s.WasmPluginType == "CPP" {
+			s.SupportedFeatures.Insert(string(WASMCPPConformanceFeature))
+		} else {
+			s.SupportedFeatures.Insert(string(WASMGoConformanceFeature))
+		}
+	} else if s.EnableAllSupportedFeatures {
+		s.SupportedFeatures = AllFeatures
+	}
+
+	for feature := range s.ExemptFeatures {
+		s.SupportedFeatures.Delete(feature)
+	}
+
 	suite := &ConformanceTestSuite{
-		Client:           s.Client,
-		RoundTripper:     roundTripper,
-		IngressClassName: s.IngressClassName,
-		Debug:            s.Debug,
-		Cleanup:          s.CleanupBaseResources,
-		BaseManifests:    s.BaseManifests,
-		GatewayAddress:   s.GatewayAddress,
+		Client:            s.Client,
+		RoundTripper:      roundTripper,
+		IngressClassName:  s.IngressClassName,
+		Debug:             s.Debug,
+		Cleanup:           s.CleanupBaseResources,
+		BaseManifests:     s.BaseManifests,
+		SupportedFeatures: s.SupportedFeatures,
+		GatewayAddress:    s.GatewayAddress,
 		Applier: kubernetes.Applier{
 			NamespaceLabels: s.NamespaceLabels,
 		},
@@ -87,20 +121,20 @@ func New(s Options) *ConformanceTestSuite {
 // Setup ensures the base resources required for conformance tests are installed
 // in the cluster. It also ensures that all relevant resources are ready.
 func (suite *ConformanceTestSuite) Setup(t *testing.T) {
-	t.Logf("Test Setup: Ensuring IngressClass has been accepted")
+	t.Logf("📦 Test Setup: Ensuring IngressClass has been accepted")
 
 	suite.Applier.IngressClass = suite.IngressClassName
 
-	t.Logf("Test Setup: Applying base manifests")
+	t.Logf("📦 Test Setup: Applying base manifests")
 	suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, suite.BaseManifests, suite.Cleanup)
 
-	t.Logf("Test Setup: Applying programmatic resources")
+	t.Logf("📦 Test Setup: Applying programmatic resources")
 	secret := kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-web-backend", "certificate", []string{"*"})
 	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
 	secret = kubernetes.MustCreateSelfSignedCertSecret(t, "higress-conformance-infra", "tls-validity-checks-certificate", []string{"*"})
 	suite.Applier.MustApplyObjectsWithCleanup(t, suite.Client, suite.TimeoutConfig, []client.Object{secret}, suite.Cleanup)
 
-	t.Logf("Test Setup: Ensuring Pods from base manifests are ready")
+	t.Logf("📦 Test Setup: Ensuring Pods from base manifests are ready")
 	namespaces := []string{
 		"higress-conformance-infra",
 		"higress-conformance-app-backend",
@@ -109,12 +143,13 @@ func (suite *ConformanceTestSuite) Setup(t *testing.T) {
 		"dubbo-demo-provider",
 	}
 	kubernetes.NamespacesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, namespaces)
+
+	t.Logf("🌱 Supported Features: %+v", suite.SupportedFeatures.UnsortedList())
 }
 
-// Run runs the provided set of conformance tests.
+// RunWithTests runs the provided set of conformance tests.
 func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
-
-	t.Logf("Start Running %d Test Cases: \n\n%s", len(tests), globalConformanceTestsListInfo(tests))
+	t.Logf("🚀 Start Running %d Test Cases: \n\n%s", len(tests), globalConformanceTestsListInfo(tests))
 	for _, test := range tests {
 		t.Run(test.ShortName, func(t *testing.T) {
 			test.Run(t, suite)
@@ -125,17 +160,20 @@ func (suite *ConformanceTestSuite) Run(t *testing.T, tests []ConformanceTest) {
 func globalConformanceTestsListInfo(tests []ConformanceTest) string {
 	var cases string
 	for index, test := range tests {
-		cases += fmt.Sprintf("CaseNum: %d\nCaseName: %s\nScenario: %s\n\n", index+1, test.ShortName, test.Description)
+		cases += fmt.Sprintf("🎯 CaseNum: %d\nCaseName: %s\nScenario: %s\nFeatures: %+v\n\n", index+1, test.ShortName, test.Description, test.Features)
 	}
 
 	return cases
 }
 
+type ConformanceTests []ConformanceTest
+
 // ConformanceTest is used to define each individual conformance test.
 type ConformanceTest struct {
 	ShortName   string
 	Description string
 	Manifests   []string
+	Features    []SupportedFeature
 	Slow        bool
 	Parallel    bool
 	Test        func(*testing.T, *ConformanceTestSuite)
@@ -148,8 +186,23 @@ func (test *ConformanceTest) Run(t *testing.T, suite *ConformanceTestSuite) {
 		t.Parallel()
 	}
 
+	// Check that all features exercised by the test have been opted into by
+	// the suite.
+	for _, feature := range test.Features {
+		if !suite.SupportedFeatures.Contains(string(feature)) {
+			t.Skipf("🏊🏼 Skipping %s: suite does not support %s", test.ShortName, feature)
+		}
+	}
+
+	// check that the test should not be skipped
+	if suite.SkipTests.Contains(test.ShortName) {
+		t.Skipf("🏊🏼 Skipping %s: test explicitly skipped", test.ShortName)
+	}
+
+	t.Logf("🔥 Running Conformance Test: %s", test.ShortName)
+
 	for _, manifestLocation := range test.Manifests {
-		t.Logf("Applying %s", manifestLocation)
+		t.Logf("🧳 Applying Manifests: %s", manifestLocation)
 		suite.Applier.MustApplyWithCleanup(t, suite.Client, suite.TimeoutConfig, manifestLocation, true)
 	}