Support configurable GatewayClass isolation (#3981)

Signed-off-by: EndlessSeeker <1766508902@qq.com>
This commit is contained in:
EndlessSeeker
2026-06-17 20:27:45 +08:00
committed by GitHub
parent e758504d72
commit efaef2e3d0
15 changed files with 252 additions and 23 deletions

View File

@@ -49,6 +49,7 @@ spec:
- --enableStatus={{ .Values.global.enableStatus }}
{{- end }}
- --ingressClass={{ .Values.global.ingressClass }}
- --gatewayClass={{ .Values.global.gatewayClass }}
{{- if .Values.global.watchNamespace }}
- --watchNamespace={{ .Values.global.watchNamespace }}
{{- end }}

View File

@@ -28,6 +28,10 @@ global:
# -- Whether to create the IngressClass resource for global.ingressClass.
# Set this to false when reusing an existing IngressClass, for example during Nginx Ingress migration.
createIngressClass: true
# -- GatewayClassName used by Higress to select Gateway API resources.
# The default value higress uses controllerName higress.io/gateway-controller.
# A custom value, for example higress-internal, uses controllerName higress.io/gateway-controller-higress-internal.
gatewayClass: "higress"
# -- If not empty, Higress Controller will only watch resources in the specified namespace.
# When isolating different business systems using K8s namespace,
# if each namespace requires a standalone gateway instance,

View File

@@ -181,6 +181,7 @@ The command removes all the Kubernetes components associated with the chart and
| global.enableSRDS | bool | `true` | |
| global.enableStatus | bool | `true` | If true, Higress Controller will update the status field of Ingress resources. When migrating from Nginx Ingress, in order to avoid status field of Ingress objects being overwritten, this parameter needs to be set to false, so Higress won't write the entry IP to the status field of the corresponding Ingress object. |
| global.externalIstiod | bool | `false` | Configure a remote cluster data plane controlled by an external istiod. When set to true, istiod is not deployed locally and only a subset of the other discovery charts are enabled. |
| global.gatewayClass | string | `"higress"` | GatewayClassName used by Higress to select Gateway API resources. The default value higress uses controllerName higress.io/gateway-controller. A custom value, for example higress-internal, uses controllerName higress.io/gateway-controller-higress-internal. |
| global.hostRDSMergeSubset | bool | `false` | |
| global.hub | string | `"higress-registry.cn-hangzhou.cr.aliyuncs.com"` | Default hub (registry) for Higress images. For Higress deployments, images are pulled from: {hub}/higress/{image} For built-in plugins, images are pulled from: {hub}/{pluginNamespace}/{plugin-name} Change this to use a mirror registry closer to your deployment region for faster image pulls. |
| global.imagePullPolicy | string | `""` | Specify image pull policy if default behavior isn't desired. Default behavior: latest images will be Always else IfNotPresent. |

View File

@@ -110,6 +110,7 @@ type ServerArgs struct {
// 2. When the ingress class is set empty, the higress controller will watch all ingress
// resources in the k8s cluster.
IngressClass string
GatewayClass string
EnableStatus bool
WatchNamespace string
GrpcKeepAliveOptions *keepalive.Options
@@ -222,6 +223,7 @@ func (s *Server) initConfigController() error {
Enable: true,
ClusterId: s.RegistryOptions.KubeOptions.ClusterID,
IngressClass: s.IngressClass,
GatewayClass: s.GatewayClass,
WatchNamespace: s.WatchNamespace,
EnableStatus: s.EnableStatus,
SystemNamespace: higressconfig.PodNamespace,

View File

@@ -106,6 +106,7 @@ func getServerCommand() *cobra.Command {
serveCmd.PersistentFlags().StringVar(&serverArgs.GatewaySelectorValue, "gatewaySelectorValue", "higress-system-higress-gateway", "gateway resource selector label value")
serveCmd.PersistentFlags().BoolVar(&serverArgs.EnableStatus, "enableStatus", true, "enable the ingress status syncer which use to update the ip in ingress's status")
serveCmd.PersistentFlags().StringVar(&serverArgs.IngressClass, "ingressClass", innerconstants.DefaultIngressClass, "if not empty, only watch the ingresses have the specified class, otherwise watch all ingresses")
serveCmd.PersistentFlags().StringVar(&serverArgs.GatewayClass, "gatewayClass", innerconstants.DefaultGatewayClass, "if not empty, only process Gateway API resources that belong to the specified GatewayClass")
serveCmd.PersistentFlags().StringVar(&serverArgs.WatchNamespace, "watchNamespace", "", "if not empty, only wath the ingresses in the specified namespace, otherwise watch in all namespacees")
serveCmd.PersistentFlags().BoolVar(&serverArgs.Debug, "debug", serverArgs.Debug, "if true, enables more debug http api")
serveCmd.PersistentFlags().StringVar(&serverArgs.HttpAddress, "httpAddress", serverArgs.HttpAddress, "the http address")

View File

@@ -77,6 +77,7 @@ func NewController(client kube.Client, options common.Options, xdsUpdater model.
ClusterID: clusterId,
Revision: higressconfig.Revision,
}
istiogateway.SetGatewayClassName(options.GatewayClass)
istioController := istiogateway.NewController(client, client.CrdWatcher().WaitForCRD, opt, xdsUpdater)
if options.GatewaySelectorKey != "" {
istioController.DefaultGatewaySelector = map[string]string{options.GatewaySelectorKey: options.GatewaySelectorValue}

View File

@@ -27,7 +27,6 @@ import (
gw "sigs.k8s.io/gateway-api/apis/v1"
gatewayx "sigs.k8s.io/gateway-api/apisx/v1alpha1"
higressconstants "github.com/alibaba/higress/v2/pkg/config/constants"
networking "istio.io/api/networking/v1alpha3"
networkingclient "istio.io/client-go/pkg/apis/networking/v1"
kubesecrets "istio.io/istio/pilot/pkg/credentials/kube"
@@ -425,7 +424,7 @@ func BackendTLSPolicyCollection(
Kind: ptr.Of(gw.Kind(gvk.KubernetesGateway.Kind)),
Name: gw.ObjectName(g.Name),
}
ancestorStatus = append(ancestorStatus, setAncestorStatus(pr, status, i.Generation, conds, gw.GatewayController(higressconstants.ManagedGatewayController)))
ancestorStatus = append(ancestorStatus, setAncestorStatus(pr, status, i.Generation, conds, gw.GatewayController(managedGatewayController)))
}
status.Ancestors = mergeAncestors(status.Ancestors, ancestorStatus)
return status, res
@@ -635,14 +634,16 @@ func parentRefEqual(a, b gw.ParentReference) bool {
ptr.Equal(a.Port, b.Port)
}
var outControllers = sets.New(gw.GatewayController(higressconstants.ManagedGatewayController), constants.ManagedGatewayMeshController)
func isOutController(controller gw.GatewayController) bool {
return controller == managedGatewayController || controller == constants.ManagedGatewayMeshController
}
// mergeAncestors merges an existing ancestor with in incoming one. We preserve order, prune stale references set by our controller,
// and add any new references from our controller.
func mergeAncestors(existing []gw.PolicyAncestorStatus, incoming []gw.PolicyAncestorStatus) []gw.PolicyAncestorStatus {
n := 0
for _, x := range existing {
if !outControllers.Contains(x.ControllerName) {
if !isOutController(x.ControllerName) {
// Keep it as-is
existing[n] = x
n++

View File

@@ -21,7 +21,6 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8s "sigs.k8s.io/gateway-api/apis/v1"
higressconstants "github.com/alibaba/higress/v2/pkg/config/constants"
"istio.io/istio/pilot/pkg/features"
"istio.io/istio/pilot/pkg/model/kstatus"
"istio.io/istio/pkg/config/schema/gvk"
@@ -49,11 +48,12 @@ func createRouteStatus(
generation int64,
currentParents []k8s.RouteParentStatus,
) []k8s.RouteParentStatus {
controllerName := k8s.GatewayController(managedGatewayController)
parents := slices.Clone(currentParents)
parentIndexes := map[string]int{}
for idx, p := range parents {
// Only consider our own
if p.ControllerName != k8s.GatewayController(higressconstants.ManagedGatewayController) {
if p.ControllerName != controllerName {
continue
}
rs := parentRefString(p.ParentRef, objectNamespace)
@@ -186,14 +186,14 @@ func createRouteStatus(
var currentConditions []metav1.Condition
currentStatus := slices.FindFunc(currentParents, func(s k8s.RouteParentStatus) bool {
return parentRefString(s.ParentRef, objectNamespace) == myRef &&
s.ControllerName == k8s.GatewayController(higressconstants.ManagedGatewayController)
s.ControllerName == controllerName
})
if currentStatus != nil {
currentConditions = currentStatus.Conditions
}
ns := k8s.RouteParentStatus{
ParentRef: gw.OriginalReference,
ControllerName: k8s.GatewayController(higressconstants.ManagedGatewayController),
ControllerName: controllerName,
Conditions: setConditions(generation, currentConditions, conds),
}
// Parent ref already exists, insert in the same place

View File

@@ -27,6 +27,7 @@ import (
)
func TestCreateRouteStatus(t *testing.T) {
setGatewayClassNameForTest(t, "")
lastTransitionTime := metav1.Now()
parentRef := httpRouteSpec.ParentRefs[0]
parentStatus := []k8s.RouteParentStatus{
@@ -122,3 +123,29 @@ func TestCreateRouteStatus(t *testing.T) {
})
}
}
func TestCreateRouteStatusWithCustomController(t *testing.T) {
if runInGatewayClassSubprocess(t) {
return
}
setGatewayClassNameForTest(t, "higress-internal")
parentRef := httpRouteSpec.ParentRefs[0]
customController := k8s.GatewayController(managedGatewayController)
current := []k8s.RouteParentStatus{
{
ParentRef: parentRef,
ControllerName: k8s.GatewayController(higressconstants.ManagedGatewayController),
},
}
got := createRouteStatus([]RouteParentResult{{OriginalReference: parentRef}}, "default", 1, current)
if len(got) != 2 {
t.Fatalf("expected default and custom controller status entries, got %+v", got)
}
if got[0].ControllerName != k8s.GatewayController(higressconstants.ManagedGatewayController) {
t.Fatalf("expected existing default controller status to be preserved, got %+v", got)
}
if got[1].ControllerName != customController {
t.Fatalf("expected custom controller status %q, got %+v", customController, got)
}
}

View File

@@ -15,6 +15,10 @@
package istio
import (
"os"
"os/exec"
"regexp"
"strings"
"testing"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -77,6 +81,7 @@ var AlwaysReady = func(class schema.GroupVersionResource, stop <-chan struct{})
}
func setupController(t *testing.T, objs ...runtime.Object) *Controller {
setGatewayClassNameForTest(t, "")
kc := kube.NewFakeClient(objs...)
setupClientCRDs(t, kc)
stop := test.NewStop(t)
@@ -94,6 +99,57 @@ func setupController(t *testing.T, objs ...runtime.Object) *Controller {
return controller
}
func setupControllerWithGatewayClass(t *testing.T, gatewayClass string, objs ...runtime.Object) *Controller {
setGatewayClassNameForTest(t, gatewayClass)
kc := kube.NewFakeClient(objs...)
setupClientCRDs(t, kc)
stop := test.NewStop(t)
controller := NewController(
kc,
AlwaysReady,
controller.Options{KrtDebugger: krt.GlobalDebugHandler},
nil)
kc.RunAndWait(stop)
go controller.Run(stop)
cg := core.NewConfigGenTest(t, core.TestOptions{})
controller.Reconcile(cg.PushContext())
kube.WaitForCacheSync("test", stop, controller.HasSynced)
return controller
}
func setGatewayClassNameForTest(t *testing.T, gatewayClass string) {
t.Helper()
if gatewayClass != "" {
SetGatewayClassName(gatewayClass)
}
}
func runInGatewayClassSubprocess(t *testing.T) bool {
t.Helper()
const env = "HIGRESS_TEST_GATEWAY_CLASS_SUBPROCESS"
if os.Getenv(env) == t.Name() {
return false
}
cmd := exec.Command(os.Args[0], "-test.run=^"+regexp.QuoteMeta(t.Name())+"$", "-test.count=1")
cmd.Env = append(testEnvWithoutCoverage(), env+"="+t.Name())
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("gateway class subprocess failed: %v\n%s", err, out)
}
return true
}
func testEnvWithoutCoverage() []string {
var out []string
for _, kv := range os.Environ() {
if strings.HasPrefix(kv, "GOCOVERDIR=") {
continue
}
out = append(out, kv)
}
return out
}
func TestListInvalidGroupVersionKind(t *testing.T) {
controller := setupController(t)
@@ -135,3 +191,52 @@ func TestListGatewayResourceType(t *testing.T) {
assert.Equal(t, c.Spec, any(expectedgw))
}
}
func TestListGatewayResourceTypeWithCustomGatewayClass(t *testing.T) {
if runInGatewayClassSubprocess(t) {
return
}
customGatewayClass := "higress-internal"
customControllerName := higressconstant.ManagedGatewayController + "-" + customGatewayClass
defaultGateway := gatewaySpec.DeepCopy()
defaultGateway.GatewayClassName = k8s.ObjectName(higressconstant.DefaultGatewayClass)
customGateway := gatewaySpec.DeepCopy()
customGateway.GatewayClassName = k8s.ObjectName(customGatewayClass)
controller := setupControllerWithGatewayClass(t, customGatewayClass,
&k8sbeta.GatewayClass{
ObjectMeta: metav1.ObjectMeta{
Name: higressconstant.DefaultGatewayClass,
},
Spec: *gatewayClassSpec,
},
&k8sbeta.GatewayClass{
ObjectMeta: metav1.ObjectMeta{
Name: customGatewayClass,
},
Spec: k8s.GatewayClassSpec{
ControllerName: k8s.GatewayController(customControllerName),
},
},
&k8sbeta.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: "default-gw",
Namespace: "ns1",
},
Spec: *defaultGateway,
},
&k8sbeta.Gateway{
ObjectMeta: metav1.ObjectMeta{
Name: "custom-gw",
Namespace: "ns1",
},
Spec: *customGateway,
})
dumpOnFailure(t, krt.GlobalDebugHandler)
cfg := controller.List(gvk.Gateway, "ns1")
assert.Equal(t, len(cfg), 1)
assert.Equal(t, cfg[0].Name, "custom-gw"+"-"+constants.KubernetesGatewayName+"-default")
assert.Equal(t, cfg[0].Namespace, "ns1")
assert.Equal(t, cfg[0].Spec, any(expectedgw))
}

View File

@@ -601,9 +601,8 @@ func init() {
features.EnableAlphaGatewayAPI = true
features.EnableAmbientWaypoints = true
features.EnableAmbientMultiNetwork = true
// Recompute with ambient enabled
classInfos = getClassInfos()
builtinClasses = getBuiltinClasses()
// Recompute with the desired feature flags.
SetGatewayClassName("")
}
type TestStatusQueue struct {

View File

@@ -48,15 +48,33 @@ type classInfo struct {
addressType gateway.AddressType
}
var classInfos = getClassInfos()
var (
gatewayClassName = gateway.ObjectName(higressconstants.DefaultGatewayClass)
managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController)
classInfos = getClassInfos()
builtinClasses = getBuiltinClasses()
)
var builtinClasses = getBuiltinClasses()
// SetGatewayClassName configures the single GatewayClassName this process owns.
func SetGatewayClassName(gatewayClass string) {
if gatewayClass == "" {
gatewayClass = higressconstants.DefaultGatewayClass
}
gatewayClassName = gateway.ObjectName(gatewayClass)
if gatewayClass == higressconstants.DefaultGatewayClass {
managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController)
} else {
managedGatewayController = gateway.GatewayController(higressconstants.ManagedGatewayController + "-" + gatewayClass)
}
classInfos = getClassInfos()
builtinClasses = getBuiltinClasses()
}
func getBuiltinClasses() map[gateway.ObjectName]gateway.GatewayController {
res := map[gateway.ObjectName]gateway.GatewayController{
// Start - Updated by Higress
//gateway.ObjectName(features.GatewayAPIDefaultGatewayClass): gateway.GatewayController(features.ManagedGatewayController),
higressconstants.DefaultGatewayClass: higressconstants.ManagedGatewayController,
gatewayClassName: managedGatewayController,
// End - Updated by Higress
}
// Start - Commented by Higress
@@ -80,8 +98,8 @@ func getBuiltinClasses() map[gateway.ObjectName]gateway.GatewayController {
func getClassInfos() map[gateway.GatewayController]classInfo {
// Start - Updated by Higress
m := map[gateway.GatewayController]classInfo{
gateway.GatewayController(higressconstants.ManagedGatewayController): {
controller: higressconstants.ManagedGatewayController,
managedGatewayController: {
controller: string(managedGatewayController),
description: "The default Higress GatewayClass",
templates: "kube-gateway",
defaultServiceType: corev1.ServiceTypeLoadBalancer,

View File

@@ -38,8 +38,11 @@ func GatewayClassesCollection(
krt.Collection[GatewayClass],
) {
return krt.NewStatusCollection(gatewayClasses, func(ctx krt.HandlerContext, obj *gateway.GatewayClass) (*gateway.GatewayClassStatus, *GatewayClass) {
_, known := classInfos[obj.Spec.ControllerName]
if !known {
if gatewayv1.ObjectName(obj.Name) != gatewayv1.ObjectName(gatewayClassName) ||
obj.Spec.ControllerName != managedGatewayController {
return nil, nil
}
if _, known := classInfos[obj.Spec.ControllerName]; !known {
return nil, nil
}
status := obj.Status.DeepCopy()

View File

@@ -16,13 +16,13 @@ package istio
import (
"fmt"
"github.com/alibaba/higress/v2/pkg/config/constants"
"testing"
"time"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
gateway "sigs.k8s.io/gateway-api/apis/v1beta1"
"github.com/alibaba/higress/v2/pkg/config/constants"
"istio.io/istio/pkg/kube"
"istio.io/istio/pkg/kube/kclient/clienttest"
"istio.io/istio/pkg/test"
@@ -30,6 +30,7 @@ import (
)
func TestClassController(t *testing.T) {
setGatewayClassNameForTest(t, "")
client := kube.NewFakeClient()
cc := NewClassController(client)
classes := clienttest.Wrap(t, cc.classes)
@@ -91,3 +92,70 @@ func TestClassController(t *testing.T) {
deleteClass("something-else")
expectClass("something-else", "")
}
func TestClassControllerWithCustomGatewayClass(t *testing.T) {
if runInGatewayClassSubprocess(t) {
return
}
gatewayClass := "higress-internal"
setGatewayClassNameForTest(t, gatewayClass)
client := kube.NewFakeClient()
controllerName := string(gateway.GatewayController(constants.ManagedGatewayController + "-" + gatewayClass))
cc := NewClassController(client)
classes := clienttest.Wrap(t, cc.classes)
stop := test.NewStop(t)
client.RunAndWait(stop)
go cc.Run(stop)
expectClass := func(name, controller string) {
t.Helper()
retry.UntilSuccessOrFail(t, func() error {
gc := classes.Get(name, "")
if controller == "" {
if gc == nil {
return nil
}
return fmt.Errorf("expected no class, got %v", gc.Spec.ControllerName)
}
if gc == nil {
return fmt.Errorf("expected class %v, got none", controller)
}
if gateway.GatewayController(controller) != gc.Spec.ControllerName {
return fmt.Errorf("expected class %v, got %v", controller, gc.Spec.ControllerName)
}
return nil
}, retry.Timeout(time.Second*3))
}
expectClass(gatewayClass, controllerName)
expectClass(constants.DefaultGatewayClass, "")
}
func TestSetGatewayClassName(t *testing.T) {
if runInGatewayClassSubprocess(t) {
return
}
SetGatewayClassName("")
if gatewayClassName != gateway.ObjectName(constants.DefaultGatewayClass) {
t.Fatalf("expected default gateway class %q, got %q", constants.DefaultGatewayClass, gatewayClassName)
}
if managedGatewayController != gateway.GatewayController(constants.ManagedGatewayController) {
t.Fatalf("expected default controller %q, got %q", constants.ManagedGatewayController, managedGatewayController)
}
customClass := "higress-internal"
SetGatewayClassName(customClass)
customController := gateway.GatewayController(constants.ManagedGatewayController + "-" + customClass)
if gatewayClassName != gateway.ObjectName(customClass) {
t.Fatalf("expected custom gateway class %q, got %q", customClass, gatewayClassName)
}
if managedGatewayController != customController {
t.Fatalf("expected custom controller %q, got %q", customController, managedGatewayController)
}
if got := builtinClasses[gateway.ObjectName(customClass)]; got != customController {
t.Fatalf("expected builtin class controller %q, got %q", customController, got)
}
if _, exists := builtinClasses[gateway.ObjectName(constants.DefaultGatewayClass)]; exists {
t.Fatalf("custom config should not include default gateway class %q", constants.DefaultGatewayClass)
}
}

View File

@@ -52,8 +52,6 @@ const (
// ControllerName is the name of this controller for labeling resources it manages
const ControllerName = "inference-controller"
var supportedControllers = getSupportedControllers()
func getSupportedControllers() sets.Set[gatewayv1.GatewayController] {
ret := sets.New[gatewayv1.GatewayController]()
for _, controller := range builtinClasses {
@@ -241,7 +239,7 @@ func findGatewayParents(
for _, parentStatus := range route.Status.Parents {
// Only consider parents managed by our supported controllers (from supportedControllers variable)
// This filters out parents from other controllers we don't manage
if !supportedControllers.Contains(parentStatus.ControllerName) {
if !getSupportedControllers().Contains(parentStatus.ControllerName) {
continue
}
@@ -354,7 +352,7 @@ func calculateAcceptedStatus(
// Check if this route has our gateway as a parent and if it's accepted
for _, parentStatus := range route.Status.Parents {
// Only consider parents managed by supported controllers
if !supportedControllers.Contains(parentStatus.ControllerName) {
if !getSupportedControllers().Contains(parentStatus.ControllerName) {
continue
}