jiazhizhong/higress
1
0
Fork 0
mirror of https://github.com/alibaba/higress.git synced 2026-02-06 23:21:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Optimize rds cache (#779)

Browse Source
This commit is contained in:
澄潭
2024-01-18 10:39:58 +08:00
committed by GitHub
parent e971faeb0b
commit ecdd077c72
3 changed files with 375 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
helm/core/values.yaml
View File

@@ -1,8 +1,8 @@
revision: ""
global:
enableSRDS: false
onDemandRDS: true
hostRDSMergeSubset: true
onDemandRDS: false
hostRDSMergeSubset: false
onlyPushRouteCluster: true
# IngressClass filters which ingress resources the higress controller watches.
# The default ingress class is higress.

373
istio/1.12/patches/istio/20240115-optimize-rds-cache.patch Normal file
View File

@@ -0,0 +1,373 @@
diff -Naur istio/pilot/pkg/model/push_context.go istio-new/pilot/pkg/model/push_context.go
--- istio/pilot/pkg/model/push_context.go 2024-01-15 20:46:45.000000000 +0800
+++ istio-new/pilot/pkg/model/push_context.go 2024-01-15 19:20:45.000000000 +0800
@@ -96,6 +96,9 @@
publicByGateway map[string][]config.Config
// root vs namespace/name ->delegate vs virtualservice gvk/namespace/name
delegates map[ConfigKey][]ConfigKey
+ // Added by ingress
+ byHost map[string][]config.Config
+ // End added by ingress
}
func newVirtualServiceIndex() virtualServiceIndex {
@@ -104,6 +107,9 @@
privateByNamespaceAndGateway: map[string]map[string][]config.Config{},
exportedToNamespaceByGateway: map[string]map[string][]config.Config{},
delegates: map[ConfigKey][]ConfigKey{},
+ // Added by ingress
+ byHost: map[string][]config.Config{},
+ // End added by ingress
}
}
@@ -857,6 +863,13 @@
return res
}
+// Added by ingress
+func (ps *PushContext) VirtualServicesForHost(proxy *Proxy, host string) []config.Config {
+ return ps.virtualServiceIndex.byHost[host]
+}
+
+// End added by ingress
+
// DelegateVirtualServicesConfigKey lists all the delegate virtual services configkeys associated with the provided virtual services
func (ps *PushContext) DelegateVirtualServicesConfigKey(vses []config.Config) []ConfigKey {
var out []ConfigKey
@@ -1468,6 +1481,11 @@
for _, virtualService := range vservices {
ns := virtualService.Namespace
rule := virtualService.Spec.(*networking.VirtualService)
+ // Added by ingress
+ for _, host := range rule.Hosts {
+ ps.virtualServiceIndex.byHost[host] = append(ps.virtualServiceIndex.byHost[host], virtualService)
+ }
+ // End added by ingress
gwNames := getGatewayNames(rule)
if len(rule.ExportTo) == 0 {
// No exportTo in virtualService. Use the global default
diff -Naur istio/pilot/pkg/networking/core/v1alpha3/gateway.go istio-new/pilot/pkg/networking/core/v1alpha3/gateway.go
--- istio/pilot/pkg/networking/core/v1alpha3/gateway.go 2024-01-15 20:46:45.000000000 +0800
+++ istio-new/pilot/pkg/networking/core/v1alpha3/gateway.go 2024-01-15 20:04:05.000000000 +0800
@@ -28,6 +28,7 @@
route "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
hcm "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"
tls "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
+ discovery "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
"github.com/hashicorp/go-multierror"
meshconfig "istio.io/api/mesh/v1alpha1"
@@ -35,6 +36,7 @@
"istio.io/istio/pilot/pkg/features"
"istio.io/istio/pilot/pkg/model"
istionetworking "istio.io/istio/pilot/pkg/networking"
+ "istio.io/istio/pilot/pkg/networking/core/v1alpha3/envoyfilter"
"istio.io/istio/pilot/pkg/networking/core/v1alpha3/extension"
"istio.io/istio/pilot/pkg/networking/core/v1alpha3/mseingress"
istio_route "istio.io/istio/pilot/pkg/networking/core/v1alpha3/route"
@@ -423,8 +425,15 @@
gatewayName string
}
-func (configgen *ConfigGeneratorImpl) buildHostRDSConfig(node *model.Proxy, push *model.PushContext,
- routeName string) *route.RouteConfiguration {
+func (configgen *ConfigGeneratorImpl) buildHostRDSConfig(
+ node *model.Proxy,
+ req *model.PushRequest,
+ routeName string,
+ vsCache map[int][]virtualServiceContext,
+ efw *model.EnvoyFilterWrapper,
+ efKeys []string,
+) (*discovery.Resource, bool) {
+ push := req.Push
var (
hostRDSPort string
hostRDSHost string
@@ -432,7 +441,7 @@
portAndHost := strings.SplitN(strings.TrimPrefix(routeName, constants.HigressHostRDSNamePrefix), ".", 2)
if len(portAndHost) != 2 {
log.Errorf("Invalid route %s when using Higress hostRDS", routeName)
- return nil
+ return nil, false
}
hostRDSPort = portAndHost[0]
hostRDSHost = portAndHost[1]
@@ -441,10 +450,24 @@
rdsPort, err := strconv.Atoi(hostRDSPort)
if err != nil {
log.Errorf("Invalid port %s of route %s when using Higress hostRDS", hostRDSPort, routeName)
- return nil
+ return nil, false
+ }
+
+ routeCache := &istio_route.Cache{
+ RouteName: routeName,
+ ProxyVersion: node.Metadata.IstioVersion,
+ ListenerPort: rdsPort,
+ // Use same host vs to cache, although the cache can be cleared when the port is different, this can be accepted
+ VirtualServices: push.VirtualServicesForHost(node, hostRDSHost),
+ EnvoyFilterKeys: efKeys,
+ }
+
+ resource, exist := configgen.Cache.Get(routeCache)
+ if exist {
+ return resource, true
}
+
listenerPort := uint32(rdsPort)
- globalHTTPFilters := mseingress.ExtractGlobalHTTPFilters(node, push)
isH3DiscoveryNeeded := false
@@ -457,9 +480,9 @@
break
}
}
-
gatewayRoutes := make(map[string]map[string][]*route.Route)
gatewayVirtualServices := make(map[string][]config.Config)
+ var listenerVirtualServices []virtualServiceContext
var selectedVirtualServices []virtualServiceContext
var vHost *route.VirtualHost
serverIterator := func(mergedServers map[model.ServerPort]*model.MergedServers) {
@@ -478,31 +501,8 @@
gatewayVirtualServices[gatewayName] = virtualServices
}
for _, virtualService := range virtualServices {
- hostMatch := false
- var selectHost string
- virtualServiceHosts := host.NewNames(virtualService.Spec.(*networking.VirtualService).Hosts)
- for _, hostname := range virtualServiceHosts {
- // exact match
- if hostname == host.Name(hostRDSHost) {
- hostMatch = true
- selectHost = hostRDSHost
- break
- }
- if features.HostRDSMergeSubset {
- // subset match
- if host.Name(hostRDSHost).SubsetOf(hostname) {
- hostMatch = true
- selectHost = string(hostname)
- }
- }
- }
- if !hostMatch {
- continue
- }
- copiedVS := virtualService.DeepCopy()
- copiedVS.Spec.(*networking.VirtualService).Hosts = []string{selectHost}
- selectedVirtualServices = append(selectedVirtualServices, virtualServiceContext{
- virtualService: copiedVS,
+ listenerVirtualServices = append(listenerVirtualServices, virtualServiceContext{
+ virtualService: virtualService,
server: server,
gatewayName: gatewayName,
})
@@ -510,15 +510,63 @@
}
}
}
- serverIterator(merged.MergedServers)
- serverIterator(merged.MergedQUICTransportServers)
- // Sort by subset
- // before: ["*.abc.com", "*.com", "www.abc.com"]
- // after: ["www.abc.com", "*.abc.com", "*.com"]
- sort.SliceStable(selectedVirtualServices, func(i, j int) bool {
- return host.Name(selectedVirtualServices[i].virtualService.Spec.(*networking.VirtualService).Hosts[0]).SubsetOf(
- host.Name(selectedVirtualServices[j].virtualService.Spec.(*networking.VirtualService).Hosts[0]))
- })
+ var vsExists bool
+ if listenerVirtualServices, vsExists = vsCache[rdsPort]; !vsExists {
+ serverIterator(merged.MergedServers)
+ serverIterator(merged.MergedQUICTransportServers)
+ vsCache[rdsPort] = listenerVirtualServices
+ }
+ for _, vsCtx := range listenerVirtualServices {
+ virtualService := vsCtx.virtualService
+ hostMatch := false
+ var selectHost string
+ for _, hostname := range virtualService.Spec.(*networking.VirtualService).Hosts {
+ // exact match
+ if hostname == hostRDSHost {
+ hostMatch = true
+ selectHost = hostRDSHost
+ break
+ }
+ if features.HostRDSMergeSubset {
+ // subset match
+ if host.Name(hostRDSHost).SubsetOf(host.Name(hostname)) {
+ hostMatch = true
+ selectHost = hostname
+ }
+ }
+ }
+ if !hostMatch {
+ continue
+ }
+ if len(virtualService.Spec.(*networking.VirtualService).Hosts) > 1 {
+ copiedVS := &networking.VirtualService{}
+ copiedVS = virtualService.Spec.(*networking.VirtualService)
+ copiedVS.Hosts = []string{selectHost}
+ selectedVirtualServices = append(selectedVirtualServices, virtualServiceContext{
+ virtualService: config.Config{
+ Meta: virtualService.Meta,
+ Spec: copiedVS,
+ Status: virtualService.Status,
+ },
+ server: vsCtx.server,
+ gatewayName: vsCtx.gatewayName,
+ })
+ } else {
+ selectedVirtualServices = append(selectedVirtualServices, vsCtx)
+ }
+ }
+ if features.HostRDSMergeSubset {
+ // Sort by subset
+ // before: ["*.abc.com", "*.com", "www.abc.com"]
+ // after: ["www.abc.com", "*.abc.com", "*.com"]
+ sort.SliceStable(selectedVirtualServices, func(i, j int) bool {
+ return host.Name(selectedVirtualServices[i].virtualService.Spec.(*networking.VirtualService).Hosts[0]).SubsetOf(
+ host.Name(selectedVirtualServices[j].virtualService.Spec.(*networking.VirtualService).Hosts[0]))
+ })
+ }
+
+ globalHTTPFilters := mseingress.ExtractGlobalHTTPFilters(node, push)
+
port := int(listenerPort)
for _, ctx := range selectedVirtualServices {
virtualService := ctx.virtualService
@@ -605,25 +653,42 @@
ValidateClusters: proto.BoolFalse,
}
- return routeCfg
+ routeCfg = envoyfilter.ApplyRouteConfigurationPatches(networking.EnvoyFilter_GATEWAY, node, efw, routeCfg)
+ resource = &discovery.Resource{
+ Name: routeName,
+ Resource: util.MessageToAny(routeCfg),
+ }
+
+ if features.EnableRDSCaching {
+ configgen.Cache.Add(routeCache, req, resource)
+ }
+
+ return resource, false
}
// End added by ingress
-func (configgen *ConfigGeneratorImpl) buildGatewayHTTPRouteConfig(node *model.Proxy, push *model.PushContext,
- routeName string) *route.RouteConfiguration {
+// Modifed by ingress
+func (configgen *ConfigGeneratorImpl) buildGatewayHTTPRouteConfig(
+ node *model.Proxy,
+ req *model.PushRequest,
+ routeName string,
+ vsCache map[int][]virtualServiceContext,
+ efw *model.EnvoyFilterWrapper,
+ efKeys []string,
+) (*discovery.Resource, bool) {
if node.MergedGateway == nil {
log.Warnf("buildGatewayRoutes: no gateways for router %v", node.ID)
- return &route.RouteConfiguration{
- Name: routeName,
- VirtualHosts: []*route.VirtualHost{},
- ValidateClusters: proto.BoolFalse,
- }
+ return nil, false
}
-
// Added by ingress
+ push := req.Push
if strings.HasPrefix(routeName, constants.HigressHostRDSNamePrefix) {
- return configgen.buildHostRDSConfig(node, push, routeName)
+ resource, cacheHit := configgen.buildHostRDSConfig(node, req, routeName, vsCache, efw, efKeys)
+ if resource == nil {
+ return nil, false
+ }
+ return resource, cacheHit
}
// End added by ingress
@@ -636,7 +701,7 @@
// This can happen when a gateway has recently been deleted. Envoy will still request route
// information due to the draining of listeners, so we should not return an error.
- return nil
+ return nil, false
}
servers := merged.ServersByRouteName[routeName]
@@ -768,9 +833,16 @@
ValidateClusters: proto.BoolFalse,
}
- return routeCfg
+ routeCfg = envoyfilter.ApplyRouteConfigurationPatches(networking.EnvoyFilter_GATEWAY, node, efw, routeCfg)
+ resource := &discovery.Resource{
+ Name: routeName,
+ Resource: util.MessageToAny(routeCfg),
+ }
+ return resource, false
}
+// End modified by ingress
+
// hashRouteList returns a hash of a list of pointers
func hashRouteList(r []*route.Route) uint64 {
hash := md5.New()
diff -Naur istio/pilot/pkg/networking/core/v1alpha3/httproute.go istio-new/pilot/pkg/networking/core/v1alpha3/httproute.go
--- istio/pilot/pkg/networking/core/v1alpha3/httproute.go 2024-01-15 20:46:41.000000000 +0800
+++ istio-new/pilot/pkg/networking/core/v1alpha3/httproute.go 2024-01-15 10:29:09.000000000 +0800
@@ -78,17 +78,30 @@
routeConfigurations = append(routeConfigurations, rc)
}
case model.Router:
+ // Modified by ingress
+ vsCache := make(map[int][]virtualServiceContext)
+ envoyfilterKeys := efw.Keys()
for _, routeName := range routeNames {
- rc := configgen.buildGatewayHTTPRouteConfig(node, req.Push, routeName)
- if rc != nil {
- rc = envoyfilter.ApplyRouteConfigurationPatches(networking.EnvoyFilter_GATEWAY, node, efw, rc)
- resource := &discovery.Resource{
+ rc, cached := configgen.buildGatewayHTTPRouteConfig(node, req, routeName, vsCache, efw, envoyfilterKeys)
+ if cached && !features.EnableUnsafeAssertions {
+ hit++
+ } else {
+ miss++
+ }
+ if rc == nil {
+ emptyRoute := &route.RouteConfiguration{
+ Name: routeName,
+ VirtualHosts: []*route.VirtualHost{},
+ ValidateClusters: proto.BoolFalse,
+ }
+ rc = &discovery.Resource{
Name: routeName,
- Resource: util.MessageToAny(rc),
+ Resource: util.MessageToAny(emptyRoute),
}
- routeConfigurations = append(routeConfigurations, resource)
}
+ routeConfigurations = append(routeConfigurations, rc)
}
+ // End modified by ingress
}
if !features.EnableRDSCaching {
return routeConfigurations, model.DefaultXdsLogDetails
diff -Naur istio/pilot/pkg/xds/discovery.go istio-new/pilot/pkg/xds/discovery.go
--- istio/pilot/pkg/xds/discovery.go 2024-01-15 20:46:45.000000000 +0800
+++ istio-new/pilot/pkg/xds/discovery.go 2024-01-12 19:56:02.000000000 +0800
@@ -392,6 +392,9 @@
// ConfigUpdate implements ConfigUpdater interface, used to request pushes.
// It replaces the 'clear cache' from v1.
func (s *DiscoveryServer) ConfigUpdate(req *model.PushRequest) {
+ if req.Full {
+ log.Infof("full push happen, reason:%v", req.Reason)
+ }
inboundConfigUpdates.Increment()
s.InboundUpdates.Inc()
s.pushChannel <- req

1
pkg/ingress/kube/common/tool.go
View File

@@ -36,7 +36,6 @@ func ValidateBackendResource(resource *v1.TypedLocalObjectReference) bool {
if resource == nil || resource.APIGroup == nil ||
*resource.APIGroup != netv1.SchemeGroupVersion.Group ||
resource.Kind != "McpBridge" || resource.Name != "default" {
IngressLog.Warnf("invalid mcpbridge resource: %v", resource)
return false
}
return true