diff --git a/pkg/ingress/kube/configmap/config.go b/pkg/ingress/kube/configmap/config.go index caf350bba..5c77289be 100644 --- a/pkg/ingress/kube/configmap/config.go +++ b/pkg/ingress/kube/configmap/config.go @@ -37,6 +37,7 @@ type HigressConfig struct { Tracing *Tracing `json:"tracing,omitempty"` Gzip *Gzip `json:"gzip,omitempty"` Downstream *Downstream `json:"downstream,omitempty"` + Upstream *Upstream `json:"upstream,omitempty"` DisableXEnvoyHeaders bool `json:"disableXEnvoyHeaders,omitempty"` AddXRealIpHeader bool `json:"addXRealIpHeader,omitempty"` } @@ -47,6 +48,7 @@ func NewDefaultHigressConfig() *HigressConfig { Tracing: NewDefaultTracing(), Gzip: NewDefaultGzip(), Downstream: globalOption.Downstream, + Upstream: globalOption.Upstream, DisableXEnvoyHeaders: globalOption.DisableXEnvoyHeaders, AddXRealIpHeader: globalOption.AddXRealIpHeader, } diff --git a/pkg/ingress/kube/configmap/global.go b/pkg/ingress/kube/configmap/global.go index 95a456a6e..f7db0d41e 100644 --- a/pkg/ingress/kube/configmap/global.go +++ b/pkg/ingress/kube/configmap/global.go @@ -38,6 +38,7 @@ const ( maxInitialConnectionWindowSize = 2147483647 defaultIdleTimeout = 180 + defaultUpStreamIdleTimeout = 10 defaultMaxRequestHeadersKb = 60 defaultConnectionBufferLimits = 32768 defaultMaxConcurrentStreams = 100 @@ -50,6 +51,7 @@ const ( // Global configures the behavior of the downstream connection, x-real-ip header and x-envoy headers. type Global struct { Downstream *Downstream `json:"downstream,omitempty"` + Upstream *Upstream `json:"upstream,omitempty"` AddXRealIpHeader bool `json:"addXRealIpHeader,omitempty"` DisableXEnvoyHeaders bool `json:"disableXEnvoyHeaders,omitempty"` } @@ -66,6 +68,12 @@ type Downstream struct { Http2 *Http2 `json:"http2,omitempty"` } +// Upstream configures the behavior of the upstream connection. +type Upstream struct { + // IdleTimeout limits the time that a connection may be idle on the upstream. + IdleTimeout uint32 `json:"idleTimeout"` +} + // Http2 configures HTTP/2 specific options. type Http2 struct { // MaxConcurrentStreams limits the number of concurrent streams allowed. @@ -124,7 +132,7 @@ func compareGlobal(old *Global, new *Global) (Result, error) { return ResultDelete, nil } - if new.Downstream == nil && !new.AddXRealIpHeader && !new.DisableXEnvoyHeaders { + if new.Downstream == nil && new.Upstream == nil && !new.AddXRealIpHeader && !new.DisableXEnvoyHeaders { return ResultDelete, nil } @@ -148,6 +156,9 @@ func deepCopyGlobal(global *Global) (*Global, error) { newGlobal.Downstream.Http2.InitialConnectionWindowSize = global.Downstream.Http2.InitialConnectionWindowSize } } + if global.Upstream != nil { + newGlobal.Upstream.IdleTimeout = global.Upstream.IdleTimeout + } newGlobal.AddXRealIpHeader = global.AddXRealIpHeader newGlobal.DisableXEnvoyHeaders = global.DisableXEnvoyHeaders return newGlobal, nil @@ -157,6 +168,7 @@ func deepCopyGlobal(global *Global) (*Global, error) { func NewDefaultGlobalOption() *Global { return &Global{ Downstream: NewDefaultDownstream(), + Upstream: NewDefaultUpStream(), AddXRealIpHeader: defaultAddXRealIpHeader, DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders, } @@ -172,6 +184,13 @@ func NewDefaultDownstream() *Downstream { } } +// NewDefaultUpStream returns a default upstream config. +func NewDefaultUpStream() *Upstream { + return &Upstream{ + IdleTimeout: defaultUpStreamIdleTimeout, + } +} + // NewDefaultHttp2 returns a default http2 config. func NewDefaultHttp2() *Http2 { return &Http2{ @@ -221,12 +240,14 @@ func (g *GlobalOptionController) GetName() string { func (g *GlobalOptionController) AddOrUpdateHigressConfig(name util.ClusterNamespacedName, old *HigressConfig, new *HigressConfig) error { newGlobal := &Global{ Downstream: new.Downstream, + Upstream: new.Upstream, AddXRealIpHeader: new.AddXRealIpHeader, DisableXEnvoyHeaders: new.DisableXEnvoyHeaders, } oldGlobal := &Global{ Downstream: old.Downstream, + Upstream: old.Upstream, AddXRealIpHeader: old.AddXRealIpHeader, DisableXEnvoyHeaders: old.DisableXEnvoyHeaders, } @@ -270,6 +291,7 @@ func (g *GlobalOptionController) ValidHigressConfig(higressConfig *HigressConfig global := &Global{ Downstream: higressConfig.Downstream, + Upstream: higressConfig.Upstream, AddXRealIpHeader: higressConfig.AddXRealIpHeader, DisableXEnvoyHeaders: higressConfig.DisableXEnvoyHeaders, } @@ -312,6 +334,17 @@ func (g *GlobalOptionController) ConstructEnvoyFilters() ([]*config.Config, erro downstreamConfig := g.generateDownstreamEnvoyFilter(downstreamStruct, bufferLimitStruct, namespace) configPatch = append(configPatch, downstreamConfig...) + if global.Upstream == nil { + return generateEnvoyFilter(namespace, configPatch), nil + } + + upstreamStruct := g.constructUpstream(global.Upstream) + if len(upstreamStruct) == 0 { + return generateEnvoyFilter(namespace, configPatch), nil + } + upstreamConfig := g.generateUpstreamEnvoyFilter(upstreamStruct, namespace) + configPatch = append(configPatch, upstreamConfig...) + return generateEnvoyFilter(namespace, configPatch), nil } @@ -371,6 +404,22 @@ func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueSt return downstreamConfig } +func (g *GlobalOptionController) generateUpstreamEnvoyFilter(upstreamValueStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { + upstreamConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{ + { + ApplyTo: networking.EnvoyFilter_CLUSTER, + Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{ + Context: networking.EnvoyFilter_GATEWAY, + }, + Patch: &networking.EnvoyFilter_Patch{ + Operation: networking.EnvoyFilter_Patch_MERGE, + Value: util.BuildPatchStruct(upstreamValueStruct), + }, + }, + } + return upstreamConfig +} + // generateAddXRealIpHeaderEnvoyFilter generates the add x-real-ip header envoy filter. func (g *GlobalOptionController) generateAddXRealIpHeaderEnvoyFilter(addXRealIpHeaderStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch { addXRealIpHeaderConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{ @@ -466,6 +515,22 @@ func (g *GlobalOptionController) constructDownstream(downstream *Downstream) str return downstreamConfig } +// constructUpstream constructs the upstream config. +func (g *GlobalOptionController) constructUpstream(upstream *Upstream) string { + upstreamConfig := "" + idleTimeout := upstream.IdleTimeout + + upstreamConfig = fmt.Sprintf(` + { + "common_http_protocol_options": { + "idleTimeout": "%ds" + } + } +`, idleTimeout) + + return upstreamConfig +} + // constructAddXRealIpHeader constructs the add x-real-ip header config. func (g *GlobalOptionController) constructAddXRealIpHeader() string { addXRealIpHeaderStruct := fmt.Sprintf(` diff --git a/pkg/ingress/kube/configmap/global_test.go b/pkg/ingress/kube/configmap/global_test.go index 0f08ad9d1..8da947075 100644 --- a/pkg/ingress/kube/configmap/global_test.go +++ b/pkg/ingress/kube/configmap/global_test.go @@ -41,6 +41,7 @@ func Test_validGlobal(t *testing.T) { name: "downstream nil", global: &Global{ Downstream: nil, + Upstream: NewDefaultUpStream(), AddXRealIpHeader: true, DisableXEnvoyHeaders: true, }, @@ -136,6 +137,9 @@ func Test_deepCopyGlobal(t *testing.T) { ConnectionBufferLimits: 4096, Http2: NewDefaultHttp2(), }, + Upstream: &Upstream{ + IdleTimeout: 10, + }, AddXRealIpHeader: true, DisableXEnvoyHeaders: true, }, @@ -146,6 +150,9 @@ func Test_deepCopyGlobal(t *testing.T) { ConnectionBufferLimits: 4096, Http2: NewDefaultHttp2(), }, + Upstream: &Upstream{ + IdleTimeout: 10, + }, AddXRealIpHeader: true, DisableXEnvoyHeaders: true, }, @@ -194,6 +201,9 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) { ConnectionBufferLimits: defaultConnectionBufferLimits, Http2: NewDefaultHttp2(), }, + Upstream: &Upstream{ + IdleTimeout: 10, + }, AddXRealIpHeader: true, DisableXEnvoyHeaders: true, }, @@ -206,6 +216,9 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) { ConnectionBufferLimits: defaultConnectionBufferLimits, Http2: NewDefaultHttp2(), }, + Upstream: &Upstream{ + IdleTimeout: 10, + }, AddXRealIpHeader: true, DisableXEnvoyHeaders: true, }, @@ -214,6 +227,7 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) { name: "delete and push", old: &HigressConfig{ Downstream: NewDefaultDownstream(), + Upstream: NewDefaultUpStream(), AddXRealIpHeader: defaultAddXRealIpHeader, DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders, }, @@ -222,6 +236,7 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) { wantEventPush: "push", wantGlobal: &Global{ Downstream: NewDefaultDownstream(), + Upstream: NewDefaultUpStream(), AddXRealIpHeader: defaultAddXRealIpHeader, DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders, }, diff --git a/test/e2e/conformance/tests/configmap-global.go b/test/e2e/conformance/tests/configmap-global.go index 2a149403e..c056ce43a 100644 --- a/test/e2e/conformance/tests/configmap-global.go +++ b/test/e2e/conformance/tests/configmap-global.go @@ -51,6 +51,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialConnectionWindowSize: 1048576, }, }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, DisableXEnvoyHeaders: true, AddXRealIpHeader: true, }, @@ -117,6 +120,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { @@ -132,6 +145,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialConnectionWindowSize: 1048576, }, }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, DisableXEnvoyHeaders: true, }, envoyAssertion: []envoy.Assertion{ @@ -190,6 +206,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { @@ -205,6 +231,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialConnectionWindowSize: 1048576, }, }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, AddXRealIpHeader: true, }, envoyAssertion: []envoy.Assertion{ @@ -268,6 +297,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { @@ -283,6 +322,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialConnectionWindowSize: 1048576, }, }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, }, envoyAssertion: []envoy.Assertion{ { @@ -338,11 +380,24 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { name: "did not set Downstream, will use default value", higressConfig: &configmap.HigressConfig{ + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, DisableXEnvoyHeaders: true, AddXRealIpHeader: true, }, @@ -405,6 +460,107 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ "idle_timeout": "180s", }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, + }, + }, + { + name: "did not set Upstream, will use default value", + higressConfig: &configmap.HigressConfig{ + Downstream: &configmap.Downstream{ + IdleTimeout: 180, + MaxRequestHeadersKb: 60, + ConnectionBufferLimits: 32768, + Http2: &configmap.Http2{ + MaxConcurrentStreams: 100, + InitialStreamWindowSize: 65535, + InitialConnectionWindowSize: 1048576, + }, + }, + DisableXEnvoyHeaders: true, + AddXRealIpHeader: true, + }, + envoyAssertion: []envoy.Assertion{ + { + Path: "configs.#.dynamic_route_configs.#.route_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "request_headers_to_add": []interface{}{ + map[string]interface{}{ + "append": false, + "header": map[string]interface{}{ + "key": "x-real-ip", + "value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%", + }, + }, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "outbound_0.0.0.0_80", + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "name": "envoy.filters.http.router", + "typed_config": map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router", + "suppress_envoy_headers": true, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "per_connection_buffer_limit_bytes": 32768, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "http2_protocol_options": map[string]interface{}{ + "max_concurrent_streams": 100, + "initial_stream_window_size": 65535, + "initial_connection_window_size": 1048576, + }, + "stream_idle_timeout": "180s", + "max_request_headers_kb": 60, + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "180s", + }, + }, + }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { @@ -486,10 +642,20 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ }, }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { - name: "did not set global config, downstream will use default value", + name: "did not set global config, downstream and upstream will use default value", higressConfig: &configmap.HigressConfig{}, envoyAssertion: []envoy.Assertion{ { @@ -541,10 +707,20 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ "idle_timeout": "180s", }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, }, }, { - name: "close the setting of idle timeout", + name: "close the setting of idle timeout in downstream", higressConfig: &configmap.HigressConfig{ Downstream: &configmap.Downstream{ IdleTimeout: 0, @@ -556,6 +732,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ InitialConnectionWindowSize: 1048576, }, }, + Upstream: &configmap.Upstream{ + IdleTimeout: 10, + }, DisableXEnvoyHeaders: true, AddXRealIpHeader: true, }, @@ -618,6 +797,110 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ "idle_timeout": "0s", }, }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "10s", + }, + }, + }, + }, + }, + { + name: "close the setting of idle timeout in upstream", + higressConfig: &configmap.HigressConfig{ + Downstream: &configmap.Downstream{ + IdleTimeout: 180, + MaxRequestHeadersKb: 60, + ConnectionBufferLimits: 32768, + Http2: &configmap.Http2{ + MaxConcurrentStreams: 100, + InitialStreamWindowSize: 65535, + InitialConnectionWindowSize: 1048576, + }, + }, + Upstream: &configmap.Upstream{ + IdleTimeout: 0, + }, + DisableXEnvoyHeaders: true, + AddXRealIpHeader: true, + }, + envoyAssertion: []envoy.Assertion{ + { + Path: "configs.#.dynamic_route_configs.#.route_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "request_headers_to_add": []interface{}{ + map[string]interface{}{ + "append": false, + "header": map[string]interface{}{ + "key": "x-real-ip", + "value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%", + }, + }, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager", + "stat_prefix": "outbound_0.0.0.0_80", + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "name": "envoy.filters.http.router", + "typed_config": map[string]interface{}{ + "@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router", + "suppress_envoy_headers": true, + }, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "per_connection_buffer_limit_bytes": 32768, + }, + }, + { + Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config", + CheckType: envoy.CheckTypeExist, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "http2_protocol_options": map[string]interface{}{ + "max_concurrent_streams": 100, + "initial_stream_window_size": 65535, + "initial_connection_window_size": 1048576, + }, + "stream_idle_timeout": "180s", + "max_request_headers_kb": 60, + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "180s", + }, + }, + }, + { + Path: "configs.#.dynamic_active_clusters.#.cluster", + CheckType: envoy.CheckTypeMatch, + TargetNamespace: "higress-system", + ExpectEnvoyConfig: map[string]interface{}{ + "common_http_protocol_options": map[string]interface{}{ + "idle_timeout": "0s", + }, + }, + }, }, }, } @@ -629,7 +912,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{ if err != nil { t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress") } - t.Logf("Checking Envoy config for test case %s", testcase.name) + t.Logf("Test Case %s", testcase.name) for _, assertion := range testcase.envoyAssertion { envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, assertion) }