From 6c7b1757b6ff0dff1bb66f3ac8e0bf795e85062a Mon Sep 17 00:00:00 2001 From: alexzzh Date: Sun, 7 Apr 2024 10:45:25 +0800 Subject: [PATCH] feat: higress global configmap support config route timeout (#883) --- pkg/ingress/kube/configmap/global.go | 43 ++++- .../e2e/conformance/tests/configmap-global.go | 170 ++++++++++++++++++ 2 files changed, 211 insertions(+), 2 deletions(-) diff --git a/pkg/ingress/kube/configmap/global.go b/pkg/ingress/kube/configmap/global.go index 886c9fc71..45c362445 100644 --- a/pkg/ingress/kube/configmap/global.go +++ b/pkg/ingress/kube/configmap/global.go @@ -38,6 +38,7 @@ const ( maxInitialConnectionWindowSize = 2147483647 defaultIdleTimeout = 180 + defaultRouteTimeout = 0 defaultUpStreamIdleTimeout = 10 defaultUpStreamConnectionBufferLimits = 10485760 defaultMaxRequestHeadersKb = 60 @@ -67,6 +68,8 @@ type Downstream struct { ConnectionBufferLimits uint32 `json:"connectionBufferLimits,omitempty"` // Http2 configures HTTP/2 specific options. Http2 *Http2 `json:"http2,omitempty"` + //RouteTimeout limits the time that timeout for the route. + RouteTimeout uint32 `json:"routeTimeout"` } // Upstream configures the behavior of the upstream connection. @@ -158,6 +161,7 @@ func deepCopyGlobal(global *Global) (*Global, error) { newGlobal.Downstream.Http2.InitialStreamWindowSize = global.Downstream.Http2.InitialStreamWindowSize newGlobal.Downstream.Http2.InitialConnectionWindowSize = global.Downstream.Http2.InitialConnectionWindowSize } + newGlobal.Downstream.RouteTimeout = global.Downstream.RouteTimeout } if global.Upstream != nil { newGlobal.Upstream.IdleTimeout = global.Upstream.IdleTimeout @@ -185,6 +189,7 @@ func NewDefaultDownstream() *Downstream { MaxRequestHeadersKb: defaultMaxRequestHeadersKb, ConnectionBufferLimits: defaultConnectionBufferLimits, Http2: NewDefaultHttp2(), + RouteTimeout: defaultRouteTimeout, } } @@ -328,7 +333,8 @@ func (g *GlobalOptionController) ConstructEnvoyFilters() ([]*config.Config, erro if global.Downstream != nil { downstreamStruct := g.constructDownstream(global.Downstream) bufferLimitStruct := g.constructBufferLimit(global.Downstream) - downstreamConfig := g.generateDownstreamEnvoyFilter(downstreamStruct, bufferLimitStruct, namespace) + routeTimeoutStruct := g.constructRouteTimeout(global.Downstream) + downstreamConfig := g.generateDownstreamEnvoyFilter(downstreamStruct, bufferLimitStruct, routeTimeoutStruct, namespace) if downstreamConfig != nil { configPatch = append(configPatch, downstreamConfig...) } @@ -371,7 +377,7 @@ func (g *GlobalOptionController) RegisterItemEventHandler(eventHandler ItemEvent } // generateDownstreamEnvoyFilter generates the downstream envoy filter. -func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueStruct string, bufferLimitStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { +func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueStruct string, bufferLimitStruct string, routeTimeoutStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { var downstreamConfig []*networking.EnvoyFilter_EnvoyConfigObjectPatch if len(downstreamValueStruct) != 0 { @@ -409,6 +415,28 @@ func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueSt }) } + if len(routeTimeoutStruct) != 0 { + downstreamConfig = append(downstreamConfig, &networking.EnvoyFilter_EnvoyConfigObjectPatch{ + ApplyTo: networking.EnvoyFilter_HTTP_ROUTE, + Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: networking.EnvoyFilter_GATEWAY, + ObjectTypes: &networking.EnvoyFilter_EnvoyConfigObjectMatch_RouteConfiguration{ + RouteConfiguration: &networking.EnvoyFilter_RouteConfigurationMatch{ + Vhost: &networking.EnvoyFilter_RouteConfigurationMatch_VirtualHostMatch{ + Route: &networking.EnvoyFilter_RouteConfigurationMatch_RouteMatch{ + Action: networking.EnvoyFilter_RouteConfigurationMatch_RouteMatch_ROUTE, + }, + }, + }, + }, + }, + Patch: &networking.EnvoyFilter_Patch{ + Operation: networking.EnvoyFilter_Patch_MERGE, + Value: util.BuildPatchStruct(routeTimeoutStruct), + }, + }) + } + return downstreamConfig } @@ -605,3 +633,14 @@ func (g *GlobalOptionController) constructBufferLimit(downstream *Downstream) st } `, downstream.ConnectionBufferLimits) } + +// constructRouteTimeout constructs the route timeout config. +func (g *GlobalOptionController) constructRouteTimeout(downstream *Downstream) string { + return fmt.Sprintf(` + { + "route": { + "timeout": "%ds" + } + } + `, downstream.RouteTimeout) +} diff --git a/test/e2e/conformance/tests/configmap-global.go b/test/e2e/conformance/tests/configmap-global.go index fa63dc26d..a64043bbb 100644 --- a/test/e2e/conformance/tests/configmap-global.go +++ b/test/e2e/conformance/tests/configmap-global.go @@ -50,6 +50,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 15, }, Upstream: &configmap.Upstream{ IdleTimeout: 10, @@ -121,6 +122,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "15s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -146,6 +155,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 15, }, Upstream: &configmap.Upstream{ IdleTimeout: 10, @@ -209,6 +219,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "15s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -234,6 +252,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 15, }, Upstream: &configmap.Upstream{ IdleTimeout: 10, @@ -302,6 +321,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "15s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -327,6 +354,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 15, }, Upstream: &configmap.Upstream{ IdleTimeout: 10, @@ -387,6 +415,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "15s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -468,6 +504,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ "idle_timeout": "180s", }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "0s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -492,6 +536,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 15, }, DisableXEnvoyHeaders: true, AddXRealIpHeader: true, @@ -559,6 +604,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "15s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -584,6 +637,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialStreamWindowSize: 65535, InitialConnectionWindowSize: 1048576, }, + RouteTimeout: 60, }, DisableXEnvoyHeaders: true, AddXRealIpHeader: true, @@ -651,6 +705,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "60s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -716,6 +778,14 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ "idle_timeout": "180s", }, }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "0s", + }, + }, { Path: "configs.#.dynamic_active_clusters.#.cluster", CheckType: envoy.CheckTypeMatch, @@ -819,6 +889,106 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + name: "close the setting of route timeout in downstream", + higressConfig: &configmap.HigressConfig{ + Downstream: &configmap.Downstream{ + IdleTimeout: 180, + MaxRequestHeadersKb: 60, + ConnectionBufferLimits: 32768, + Http2: &configmap.Http2{ + MaxConcurrentStreams: 100, + InitialStreamWindowSize: 65535, + InitialConnectionWindowSize: 1048576, + }, + RouteTimeout: 0, + }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, + DisableXEnvoyHeaders: true, + AddXRealIpHeader: true, + }, + envoyAssertion: []envoy.Assertion{ + { + Path: "configs.#.dynamic_route_configs.#.route_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "request_headers_to_add": []interface{}{ + map[string]interface{}{ + "append": false, + "header": map[string]interface{}{ + "key": "x-real-ip", + "value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%", + }, + }, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "outbound_0.0.0.0_80", + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "name": "envoy.filters.http.router", + "typed_config": map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router", + "suppress_envoy_headers": true, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "per_connection_buffer_limit_bytes": 32768, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "max_concurrent_streams": 100, + "initial_stream_window_size": 65535, + "initial_connection_window_size": 1048576, + "stream_idle_timeout": "180s", + "max_request_headers_kb": 60, + "idle_timeout": "180s", + }, + }, + { + Path: "configs.#.dynamic_route_configs.#.route_config.virtual_hosts.#.routes.#.route", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "timeout": "0s", + }, + }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + "per_connection_buffer_limit_bytes": 10485760, + }, + }, + }, + }, { name: "close the setting of idle timeout in upstream", higressConfig: &configmap.HigressConfig{