Support configurable GatewayClass isolation (#3981)

Signed-off-by: EndlessSeeker <1766508902@qq.com>

pkg/bootstrap/server.go

@@ -110,6 +110,7 @@ type ServerArgs struct {
 	// 2. When the ingress class is set empty, the higress controller will watch all ingress
 	// resources in the k8s cluster.
 	IngressClass         string
+	GatewayClass         string
 	EnableStatus         bool
 	WatchNamespace       string
 	GrpcKeepAliveOptions *keepalive.Options
@@ -222,6 +223,7 @@ func (s *Server) initConfigController() error {
 		Enable:               true,
 		ClusterId:            s.RegistryOptions.KubeOptions.ClusterID,
 		IngressClass:         s.IngressClass,
+		GatewayClass:         s.GatewayClass,
 		WatchNamespace:       s.WatchNamespace,
 		EnableStatus:         s.EnableStatus,
 		SystemNamespace:      higressconfig.PodNamespace,

pkg/cmd/server.go

@@ -106,6 +106,7 @@ func getServerCommand() *cobra.Command {
 	serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorValue, "gatewaySelectorValue", "higress-system-higress-gateway", "gateway resource selector label value")
 	serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableStatus, "enableStatus", true, "enable the ingress status syncer which use to update the ip in ingress's status")
 	serveCmd.PersistentFlags().StringVar(&serverArgs.IngressClass, "ingressClass", innerconstants.DefaultIngressClass, "if not empty, only watch the ingresses have the specified class, otherwise watch all ingresses")
+	serveCmd.PersistentFlags().StringVar(&serverArgs.GatewayClass, "gatewayClass", innerconstants.DefaultGatewayClass, "if not empty, only process Gateway API resources that belong to the specified GatewayClass")
 	serveCmd.PersistentFlags().StringVar(&serverArgs.WatchNamespace, "watchNamespace", "", "if not empty, only wath the ingresses in the specified namespace, otherwise watch in all namespacees")
 	serveCmd.PersistentFlags().BoolVar(&serverArgs.Debug, "debug", serverArgs.Debug, "if true, enables more debug http api")
 	serveCmd.PersistentFlags().StringVar(&serverArgs.HttpAddress, "httpAddress", serverArgs.HttpAddress, "the http address")

pkg/ingress/kube/gateway/controller.go

@@ -77,6 +77,7 @@ func NewController(client kube.Client, options common.Options, xdsUpdater model.
 		ClusterID:    clusterId,
 		Revision:     higressconfig.Revision,
 	}
+	istiogateway.SetGatewayClassName(options.GatewayClass)
 	istioController := istiogateway.NewController(client, client.CrdWatcher().WaitForCRD, opt, xdsUpdater)
 	if options.GatewaySelectorKey != "" {
 		istioController.DefaultGatewaySelector = map[string]string{options.GatewaySelectorKey: options.GatewaySelectorValue}

pkg/ingress/kube/gateway/istio/backend_policies.go

@@ -27,7 +27,6 @@ import (
 	gw "sigs.k8s.io/gateway-api/apis/v1"
 	gatewayx "sigs.k8s.io/gateway-api/apisx/v1alpha1"
 
-	higressconstants "github.com/alibaba/higress/v2/pkg/config/constants"
 	networking "istio.io/api/networking/v1alpha3"
 	networkingclient "istio.io/client-go/pkg/apis/networking/v1"
 	kubesecrets "istio.io/istio/pilot/pkg/credentials/kube"
@@ -425,7 +424,7 @@ func BackendTLSPolicyCollection(
 				Kind:  ptr.Of(gw.Kind(gvk.KubernetesGateway.Kind)),
 				Name:  gw.ObjectName(g.Name),
 			}
-			ancestorStatus = append(ancestorStatus, setAncestorStatus(pr, status, i.Generation, conds, gw.GatewayController(higressconstants.ManagedGatewayController)))
+			ancestorStatus = append(ancestorStatus, setAncestorStatus(pr, status, i.Generation, conds, gw.GatewayController(managedGatewayController)))
 		}
 		status.Ancestors = mergeAncestors(status.Ancestors, ancestorStatus)
 		return status, res
@@ -635,14 +634,16 @@ func parentRefEqual(a, b gw.ParentReference) bool {
 		ptr.Equal(a.Port, b.Port)
 }
 
-var outControllers = sets.New(gw.GatewayController(higressconstants.ManagedGatewayController), constants.ManagedGatewayMeshController)
+func isOutController(controller gw.GatewayController) bool {
+	return controller == managedGatewayController || controller == constants.ManagedGatewayMeshController
+}
 
 // mergeAncestors merges an existing ancestor with in incoming one. We preserve order, prune stale references set by our controller,
 // and add any new references from our controller.
 func mergeAncestors(existing []gw.PolicyAncestorStatus, incoming []gw.PolicyAncestorStatus) []gw.PolicyAncestorStatus {
 	n := 0
 	for _, x := range existing {
-		if !outControllers.Contains(x.ControllerName) {
+		if !isOutController(x.ControllerName) {
 			// Keep it as-is
 			existing[n] = x
 			n++

pkg/ingress/kube/gateway/istio/conditions.go

@@ -21,7 +21,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8s "sigs.k8s.io/gateway-api/apis/v1"
 
-	higressconstants "github.com/alibaba/higress/v2/pkg/config/constants"
 	"istio.io/istio/pilot/pkg/features"
 	"istio.io/istio/pilot/pkg/model/kstatus"
 	"istio.io/istio/pkg/config/schema/gvk"
@@ -49,11 +48,12 @@ func createRouteStatus(
 	generation int64,
 	currentParents []k8s.RouteParentStatus,
 ) []k8s.RouteParentStatus {
+	controllerName := k8s.GatewayController(managedGatewayController)
 	parents := slices.Clone(currentParents)
 	parentIndexes := map[string]int{}
 	for idx, p := range parents {
 		// Only consider our own
-		if p.ControllerName != k8s.GatewayController(higressconstants.ManagedGatewayController) {
+		if p.ControllerName != controllerName {
 			continue
 		}
 		rs := parentRefString(p.ParentRef, objectNamespace)
@@ -186,14 +186,14 @@ func createRouteStatus(
 		var currentConditions []metav1.Condition
 		currentStatus := slices.FindFunc(currentParents, func(s k8s.RouteParentStatus) bool {
 			return parentRefString(s.ParentRef, objectNamespace) == myRef &&
-				s.ControllerName == k8s.GatewayController(higressconstants.ManagedGatewayController)
+				s.ControllerName == controllerName
 		})
 		if currentStatus != nil {
 			currentConditions = currentStatus.Conditions
 		}
 		ns := k8s.RouteParentStatus{
 			ParentRef:      gw.OriginalReference,
-			ControllerName: k8s.GatewayController(higressconstants.ManagedGatewayController),
+			ControllerName: controllerName,
 			Conditions:     setConditions(generation, currentConditions, conds),
 		}
 		// Parent ref already exists, insert in the same place

pkg/ingress/kube/gateway/istio/conditions_test.go

@@ -27,6 +27,7 @@ import (
 )
 
 func TestCreateRouteStatus(t *testing.T) {
+	setGatewayClassNameForTest(t, "")
 	lastTransitionTime := metav1.Now()
 	parentRef := httpRouteSpec.ParentRefs[0]
 	parentStatus := []k8s.RouteParentStatus{
@@ -122,3 +123,29 @@ func TestCreateRouteStatus(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateRouteStatusWithCustomController(t *testing.T) {
+	if runInGatewayClassSubprocess(t) {
+		return
+	}
+	setGatewayClassNameForTest(t, "higress-internal")
+	parentRef := httpRouteSpec.ParentRefs[0]
+	customController := k8s.GatewayController(managedGatewayController)
+	current := []k8s.RouteParentStatus{
+		{
+			ParentRef:      parentRef,
+			ControllerName: k8s.GatewayController(higressconstants.ManagedGatewayController),
+		},
+	}
+
+	got := createRouteStatus([]RouteParentResult{{OriginalReference: parentRef}}, "default", 1, current)
+	if len(got) != 2 {
+		t.Fatalf("expected default and custom controller status entries, got %+v", got)
+	}
+	if got[0].ControllerName != k8s.GatewayController(higressconstants.ManagedGatewayController) {
+		t.Fatalf("expected existing default controller status to be preserved, got %+v", got)
+	}
+	if got[1].ControllerName != customController {
+		t.Fatalf("expected custom controller status %q, got %+v", customController, got)
+	}
+}

pkg/ingress/kube/gateway/istio/controller_test.go

@@ -15,6 +15,10 @@
 package istio
 
 import (
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
 	"testing"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -77,6 +81,7 @@ var AlwaysReady = func(class schema.GroupVersionResource, stop <-chan struct{})
 }
 
 func setupController(t *testing.T, objs ...runtime.Object) *Controller {
+	setGatewayClassNameForTest(t, "")
 	kc := kube.NewFakeClient(objs...)
 	setupClientCRDs(t, kc)
 	stop := test.NewStop(t)
@@ -94,6 +99,57 @@ func setupController(t *testing.T, objs ...runtime.Object) *Controller {
 	return controller
 }
 
+func setupControllerWithGatewayClass(t *testing.T, gatewayClass string, objs ...runtime.Object) *Controller {
+	setGatewayClassNameForTest(t, gatewayClass)
+	kc := kube.NewFakeClient(objs...)
+	setupClientCRDs(t, kc)
+	stop := test.NewStop(t)
+	controller := NewController(
+		kc,
+		AlwaysReady,
+		controller.Options{KrtDebugger: krt.GlobalDebugHandler},
+		nil)
+	kc.RunAndWait(stop)
+	go controller.Run(stop)
+	cg := core.NewConfigGenTest(t, core.TestOptions{})
+	controller.Reconcile(cg.PushContext())
+	kube.WaitForCacheSync("test", stop, controller.HasSynced)
+
+	return controller
+}
+
+func setGatewayClassNameForTest(t *testing.T, gatewayClass string) {
+	t.Helper()
+	if gatewayClass != "" {
+		SetGatewayClassName(gatewayClass)
+	}
+}
+
+func runInGatewayClassSubprocess(t *testing.T) bool {
+	t.Helper()
+	const env = "HIGRESS_TEST_GATEWAY_CLASS_SUBPROCESS"
+	if os.Getenv(env) == t.Name() {
+		return false
+	}
+	cmd := exec.Command(os.Args[0], "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", "-test.count=1")
+	cmd.Env = append(testEnvWithoutCoverage(), env+"="+t.Name())
+	if out, err := cmd.CombinedOutput(); err != nil {
+		t.Fatalf("gateway class subprocess failed: %v\n%s", err, out)
+	}
+	return true
+}
+
+func testEnvWithoutCoverage() []string {
+	var out []string
+	for _, kv := range os.Environ() {
+		if strings.HasPrefix(kv, "GOCOVERDIR=") {
+			continue
+		}
+		out = append(out, kv)
+	}
+	return out
+}
+
 func TestListInvalidGroupVersionKind(t *testing.T) {
 	controller := setupController(t)
 
@@ -135,3 +191,52 @@ func TestListGatewayResourceType(t *testing.T) {
 		assert.Equal(t, c.Spec, any(expectedgw))
 	}
 }
+
+func TestListGatewayResourceTypeWithCustomGatewayClass(t *testing.T) {
+	if runInGatewayClassSubprocess(t) {
+		return
+	}
+	customGatewayClass := "higress-internal"
+	customControllerName := higressconstant.ManagedGatewayController + "-" + customGatewayClass
+	defaultGateway := gatewaySpec.DeepCopy()
+	defaultGateway.GatewayClassName = k8s.ObjectName(higressconstant.DefaultGatewayClass)
+	customGateway := gatewaySpec.DeepCopy()
+	customGateway.GatewayClassName = k8s.ObjectName(customGatewayClass)
+
+	controller := setupControllerWithGatewayClass(t, customGatewayClass,
+		&k8sbeta.GatewayClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: higressconstant.DefaultGatewayClass,
+			},
+			Spec: *gatewayClassSpec,
+		},
+		&k8sbeta.GatewayClass{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: customGatewayClass,
+			},
+			Spec: k8s.GatewayClassSpec{
+				ControllerName: k8s.GatewayController(customControllerName),
+			},
+		},
+		&k8sbeta.Gateway{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "default-gw",
+				Namespace: "ns1",
+			},
+			Spec: *defaultGateway,
+		},
+		&k8sbeta.Gateway{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "custom-gw",
+				Namespace: "ns1",
+			},
+			Spec: *customGateway,
+		})
+
+	dumpOnFailure(t, krt.GlobalDebugHandler)
+	cfg := controller.List(gvk.Gateway, "ns1")
+	assert.Equal(t, len(cfg), 1)
+	assert.Equal(t, cfg[0].Name, "custom-gw"+"-"+constants.KubernetesGatewayName+"-default")
+	assert.Equal(t, cfg[0].Namespace, "ns1")
+	assert.Equal(t, cfg[0].Spec, any(expectedgw))
+}

pkg/ingress/kube/gateway/istio/conversion_test.go

@@ -601,9 +601,8 @@ func init() {
 	features.EnableAlphaGatewayAPI = true
 	features.EnableAmbientWaypoints = true
 	features.EnableAmbientMultiNetwork = true
-	// Recompute with ambient enabled
-	classInfos = getClassInfos()
-	builtinClasses = getBuiltinClasses()
+	// Recompute with the desired feature flags.
+	SetGatewayClassName("")
 }
 
 type TestStatusQueue struct {

pkg/ingress/kube/gateway/istio/deploymentcontroller.go

@@ -48,15 +48,33 @@ type classInfo struct {
 	addressType gateway.AddressType
 }
 
-var classInfos = getClassInfos()
+var (
+	gatewayClassName         = gateway.ObjectName(higressconstants.DefaultGatewayClass)
+	managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController)
+	classInfos               = getClassInfos()
+	builtinClasses           = getBuiltinClasses()
+)
 
-var builtinClasses = getBuiltinClasses()
+// SetGatewayClassName configures the single GatewayClassName this process owns.
+func SetGatewayClassName(gatewayClass string) {
+	if gatewayClass == "" {
+		gatewayClass = higressconstants.DefaultGatewayClass
+	}
+	gatewayClassName = gateway.ObjectName(gatewayClass)
+	if gatewayClass == higressconstants.DefaultGatewayClass {
+		managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController)
+	} else {
+		managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController + "-" + gatewayClass)
+	}
+	classInfos = getClassInfos()
+	builtinClasses = getBuiltinClasses()
+}
 
 func getBuiltinClasses() map[gateway.ObjectName]gateway.GatewayController {
 	res := map[gateway.ObjectName]gateway.GatewayController{
 		// Start - Updated by Higress
 		//gateway.ObjectName(features.GatewayAPIDefaultGatewayClass): gateway.GatewayController(features.ManagedGatewayController),
-		higressconstants.DefaultGatewayClass: higressconstants.ManagedGatewayController,
+		gatewayClassName: managedGatewayController,
 		// End - Updated by Higress
 	}
 	// Start - Commented by Higress
@@ -80,8 +98,8 @@ func getBuiltinClasses() map[gateway.ObjectName]gateway.GatewayController {
 func getClassInfos() map[gateway.GatewayController]classInfo {
 	// Start - Updated by Higress
 	m := map[gateway.GatewayController]classInfo{
-		gateway.GatewayController(higressconstants.ManagedGatewayController): {
-			controller:         higressconstants.ManagedGatewayController,
+		managedGatewayController: {
+			controller:         string(managedGatewayController),
 			description:        "The default Higress GatewayClass",
 			templates:          "kube-gateway",
 			defaultServiceType: corev1.ServiceTypeLoadBalancer,

pkg/ingress/kube/gateway/istio/gatewayclass_collection.go

@@ -38,8 +38,11 @@ func GatewayClassesCollection(
 	krt.Collection[GatewayClass],
 ) {
 	return krt.NewStatusCollection(gatewayClasses, func(ctx krt.HandlerContext, obj *gateway.GatewayClass) (*gateway.GatewayClassStatus, *GatewayClass) {
-		_, known := classInfos[obj.Spec.ControllerName]
-		if !known {
+		if gatewayv1.ObjectName(obj.Name) != gatewayv1.ObjectName(gatewayClassName) ||
+			obj.Spec.ControllerName != managedGatewayController {
+			return nil, nil
+		}
+		if _, known := classInfos[obj.Spec.ControllerName]; !known {
 			return nil, nil
 		}
 		status := obj.Status.DeepCopy()

pkg/ingress/kube/gateway/istio/gatewayclass_test.go

@@ -16,13 +16,13 @@ package istio
 
 import (
 	"fmt"
-	"github.com/alibaba/higress/v2/pkg/config/constants"
 	"testing"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	gateway "sigs.k8s.io/gateway-api/apis/v1beta1"
 
+	"github.com/alibaba/higress/v2/pkg/config/constants"
 	"istio.io/istio/pkg/kube"
 	"istio.io/istio/pkg/kube/kclient/clienttest"
 	"istio.io/istio/pkg/test"
@@ -30,6 +30,7 @@ import (
 )
 
 func TestClassController(t *testing.T) {
+	setGatewayClassNameForTest(t, "")
 	client := kube.NewFakeClient()
 	cc := NewClassController(client)
 	classes := clienttest.Wrap(t, cc.classes)
@@ -91,3 +92,70 @@ func TestClassController(t *testing.T) {
 	deleteClass("something-else")
 	expectClass("something-else", "")
 }
+
+func TestClassControllerWithCustomGatewayClass(t *testing.T) {
+	if runInGatewayClassSubprocess(t) {
+		return
+	}
+	gatewayClass := "higress-internal"
+	setGatewayClassNameForTest(t, gatewayClass)
+	client := kube.NewFakeClient()
+	controllerName := string(gateway.GatewayController(constants.ManagedGatewayController + "-" + gatewayClass))
+	cc := NewClassController(client)
+	classes := clienttest.Wrap(t, cc.classes)
+	stop := test.NewStop(t)
+	client.RunAndWait(stop)
+	go cc.Run(stop)
+
+	expectClass := func(name, controller string) {
+		t.Helper()
+		retry.UntilSuccessOrFail(t, func() error {
+			gc := classes.Get(name, "")
+			if controller == "" {
+				if gc == nil {
+					return nil
+				}
+				return fmt.Errorf("expected no class, got %v", gc.Spec.ControllerName)
+			}
+			if gc == nil {
+				return fmt.Errorf("expected class %v, got none", controller)
+			}
+			if gateway.GatewayController(controller) != gc.Spec.ControllerName {
+				return fmt.Errorf("expected class %v, got %v", controller, gc.Spec.ControllerName)
+			}
+			return nil
+		}, retry.Timeout(time.Second*3))
+	}
+
+	expectClass(gatewayClass, controllerName)
+	expectClass(constants.DefaultGatewayClass, "")
+}
+
+func TestSetGatewayClassName(t *testing.T) {
+	if runInGatewayClassSubprocess(t) {
+		return
+	}
+	SetGatewayClassName("")
+	if gatewayClassName != gateway.ObjectName(constants.DefaultGatewayClass) {
+		t.Fatalf("expected default gateway class %q, got %q", constants.DefaultGatewayClass, gatewayClassName)
+	}
+	if managedGatewayController != gateway.GatewayController(constants.ManagedGatewayController) {
+		t.Fatalf("expected default controller %q, got %q", constants.ManagedGatewayController, managedGatewayController)
+	}
+
+	customClass := "higress-internal"
+	SetGatewayClassName(customClass)
+	customController := gateway.GatewayController(constants.ManagedGatewayController + "-" + customClass)
+	if gatewayClassName != gateway.ObjectName(customClass) {
+		t.Fatalf("expected custom gateway class %q, got %q", customClass, gatewayClassName)
+	}
+	if managedGatewayController != customController {
+		t.Fatalf("expected custom controller %q, got %q", customController, managedGatewayController)
+	}
+	if got := builtinClasses[gateway.ObjectName(customClass)]; got != customController {
+		t.Fatalf("expected builtin class controller %q, got %q", customController, got)
+	}
+	if _, exists := builtinClasses[gateway.ObjectName(constants.DefaultGatewayClass)]; exists {
+		t.Fatalf("custom config should not include default gateway class %q", constants.DefaultGatewayClass)
+	}
+}

pkg/ingress/kube/gateway/istio/inferencepool_collection.go

@@ -52,8 +52,6 @@ const (
 // ControllerName is the name of this controller for labeling resources it manages
 const ControllerName = "inference-controller"
 
-var supportedControllers = getSupportedControllers()
-
 func getSupportedControllers() sets.Set[gatewayv1.GatewayController] {
 	ret := sets.New[gatewayv1.GatewayController]()
 	for _, controller := range builtinClasses {
@@ -241,7 +239,7 @@ func findGatewayParents(
 		for _, parentStatus := range route.Status.Parents {
 			// Only consider parents managed by our supported controllers (from supportedControllers variable)
 			// This filters out parents from other controllers we don't manage
-			if !supportedControllers.Contains(parentStatus.ControllerName) {
+			if !getSupportedControllers().Contains(parentStatus.ControllerName) {
 				continue
 			}
 
@@ -354,7 +352,7 @@ func calculateAcceptedStatus(
 		// Check if this route has our gateway as a parent and if it's accepted
 		for _, parentStatus := range route.Status.Parents {
 			// Only consider parents managed by supported controllers
-			if !supportedControllers.Contains(parentStatus.ControllerName) {
+			if !getSupportedControllers().Contains(parentStatus.ControllerName) {
 				continue
 			}