jiazhizhong/higress
1
0
Fork 0
mirror of https://github.com/alibaba/higress.git synced 2026-03-20 02:07:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add global option for upstream (#798)

Browse Source 
Signed-off-by: sjcsjc123 <1401189096@qq.com>
This commit is contained in:
SJC
2024-01-26 11:25:59 +08:00
committed by GitHub
parent 073c10df77
commit acd80d2528
4 changed files with 369 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/ingress/kube/configmap/config.go
View File

@@ -37,6 +37,7 @@ type HigressConfig struct {
Tracing *Tracing `json:"tracing,omitempty"`
Gzip *Gzip `json:"gzip,omitempty"`
Downstream *Downstream `json:"downstream,omitempty"`
Upstream *Upstream `json:"upstream,omitempty"`
DisableXEnvoyHeaders bool `json:"disableXEnvoyHeaders,omitempty"`
AddXRealIpHeader bool `json:"addXRealIpHeader,omitempty"`
}
@@ -47,6 +48,7 @@ func NewDefaultHigressConfig() *HigressConfig {
Tracing: NewDefaultTracing(),
Gzip: NewDefaultGzip(),
Downstream: globalOption.Downstream,
Upstream: globalOption.Upstream,
DisableXEnvoyHeaders: globalOption.DisableXEnvoyHeaders,
AddXRealIpHeader: globalOption.AddXRealIpHeader,
}

67
pkg/ingress/kube/configmap/global.go
View File

@@ -38,6 +38,7 @@ const (
maxInitialConnectionWindowSize = 2147483647
defaultIdleTimeout = 180
defaultUpStreamIdleTimeout = 10
defaultMaxRequestHeadersKb = 60
defaultConnectionBufferLimits = 32768
defaultMaxConcurrentStreams = 100
@@ -50,6 +51,7 @@ const (
// Global configures the behavior of the downstream connection, x-real-ip header and x-envoy headers.
type Global struct {
Downstream *Downstream `json:"downstream,omitempty"`
Upstream *Upstream `json:"upstream,omitempty"`
AddXRealIpHeader bool `json:"addXRealIpHeader,omitempty"`
DisableXEnvoyHeaders bool `json:"disableXEnvoyHeaders,omitempty"`
}
@@ -66,6 +68,12 @@ type Downstream struct {
Http2 *Http2 `json:"http2,omitempty"`
}
// Upstream configures the behavior of the upstream connection.
type Upstream struct {
// IdleTimeout limits the time that a connection may be idle on the upstream.
IdleTimeout uint32 `json:"idleTimeout"`
}
// Http2 configures HTTP/2 specific options.
type Http2 struct {
// MaxConcurrentStreams limits the number of concurrent streams allowed.
@@ -124,7 +132,7 @@ func compareGlobal(old *Global, new *Global) (Result, error) {
return ResultDelete, nil
}
if new.Downstream == nil && !new.AddXRealIpHeader && !new.DisableXEnvoyHeaders {
if new.Downstream == nil && new.Upstream == nil && !new.AddXRealIpHeader && !new.DisableXEnvoyHeaders {
return ResultDelete, nil
}
@@ -148,6 +156,9 @@ func deepCopyGlobal(global *Global) (*Global, error) {
newGlobal.Downstream.Http2.InitialConnectionWindowSize = global.Downstream.Http2.InitialConnectionWindowSize
}
}
if global.Upstream != nil {
newGlobal.Upstream.IdleTimeout = global.Upstream.IdleTimeout
}
newGlobal.AddXRealIpHeader = global.AddXRealIpHeader
newGlobal.DisableXEnvoyHeaders = global.DisableXEnvoyHeaders
return newGlobal, nil
@@ -157,6 +168,7 @@ func deepCopyGlobal(global *Global) (*Global, error) {
func NewDefaultGlobalOption() *Global {
return &Global{
Downstream: NewDefaultDownstream(),
Upstream: NewDefaultUpStream(),
AddXRealIpHeader: defaultAddXRealIpHeader,
DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders,
}
@@ -172,6 +184,13 @@ func NewDefaultDownstream() *Downstream {
}
}
// NewDefaultUpStream returns a default upstream config.
func NewDefaultUpStream() *Upstream {
return &Upstream{
IdleTimeout: defaultUpStreamIdleTimeout,
}
}
// NewDefaultHttp2 returns a default http2 config.
func NewDefaultHttp2() *Http2 {
return &Http2{
@@ -221,12 +240,14 @@ func (g *GlobalOptionController) GetName() string {
func (g *GlobalOptionController) AddOrUpdateHigressConfig(name util.ClusterNamespacedName, old *HigressConfig, new *HigressConfig) error {
newGlobal := &Global{
Downstream: new.Downstream,
Upstream: new.Upstream,
AddXRealIpHeader: new.AddXRealIpHeader,
DisableXEnvoyHeaders: new.DisableXEnvoyHeaders,
}
oldGlobal := &Global{
Downstream: old.Downstream,
Upstream: old.Upstream,
AddXRealIpHeader: old.AddXRealIpHeader,
DisableXEnvoyHeaders: old.DisableXEnvoyHeaders,
}
@@ -270,6 +291,7 @@ func (g *GlobalOptionController) ValidHigressConfig(higressConfig *HigressConfig
global := &Global{
Downstream: higressConfig.Downstream,
Upstream: higressConfig.Upstream,
AddXRealIpHeader: higressConfig.AddXRealIpHeader,
DisableXEnvoyHeaders: higressConfig.DisableXEnvoyHeaders,
}
@@ -312,6 +334,17 @@ func (g *GlobalOptionController) ConstructEnvoyFilters() ([]*config.Config, erro
downstreamConfig := g.generateDownstreamEnvoyFilter(downstreamStruct, bufferLimitStruct, namespace)
configPatch = append(configPatch, downstreamConfig...)
if global.Upstream == nil {
return generateEnvoyFilter(namespace, configPatch), nil
}
upstreamStruct := g.constructUpstream(global.Upstream)
if len(upstreamStruct) == 0 {
return generateEnvoyFilter(namespace, configPatch), nil
}
upstreamConfig := g.generateUpstreamEnvoyFilter(upstreamStruct, namespace)
configPatch = append(configPatch, upstreamConfig...)
return generateEnvoyFilter(namespace, configPatch), nil
}
@@ -371,6 +404,22 @@ func (g *GlobalOptionController) generateDownstreamEnvoyFilter(downstreamValueSt
return downstreamConfig
}
func (g *GlobalOptionController) generateUpstreamEnvoyFilter(upstreamValueStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch {
upstreamConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{
{
ApplyTo: networking.EnvoyFilter_CLUSTER,
Match: &networking.EnvoyFilter_EnvoyConfigObjectMatch{
Context: networking.EnvoyFilter_GATEWAY,
},
Patch: &networking.EnvoyFilter_Patch{
Operation: networking.EnvoyFilter_Patch_MERGE,
Value: util.BuildPatchStruct(upstreamValueStruct),
},
},
}
return upstreamConfig
}
// generateAddXRealIpHeaderEnvoyFilter generates the add x-real-ip header envoy filter.
func (g *GlobalOptionController) generateAddXRealIpHeaderEnvoyFilter(addXRealIpHeaderStruct string, namespace string) []*networking.EnvoyFilter_EnvoyConfigObjectPatch {
addXRealIpHeaderConfig := []*networking.EnvoyFilter_EnvoyConfigObjectPatch{
@@ -466,6 +515,22 @@ func (g *GlobalOptionController) constructDownstream(downstream *Downstream) str
return downstreamConfig
}
// constructUpstream constructs the upstream config.
func (g *GlobalOptionController) constructUpstream(upstream *Upstream) string {
upstreamConfig := ""
idleTimeout := upstream.IdleTimeout
upstreamConfig = fmt.Sprintf(`
{
"common_http_protocol_options": {
"idleTimeout": "%ds"
}
}
`, idleTimeout)
return upstreamConfig
}
// constructAddXRealIpHeader constructs the add x-real-ip header config.
func (g *GlobalOptionController) constructAddXRealIpHeader() string {
addXRealIpHeaderStruct := fmt.Sprintf(`

15
pkg/ingress/kube/configmap/global_test.go
View File

@@ -41,6 +41,7 @@ func Test_validGlobal(t *testing.T) {
name: "downstream nil",
global: &Global{
Downstream: nil,
Upstream: NewDefaultUpStream(),
AddXRealIpHeader: true,
DisableXEnvoyHeaders: true,
},
@@ -136,6 +137,9 @@ func Test_deepCopyGlobal(t *testing.T) {
ConnectionBufferLimits: 4096,
Http2: NewDefaultHttp2(),
},
Upstream: &Upstream{
IdleTimeout: 10,
},
AddXRealIpHeader: true,
DisableXEnvoyHeaders: true,
},
@@ -146,6 +150,9 @@ func Test_deepCopyGlobal(t *testing.T) {
ConnectionBufferLimits: 4096,
Http2: NewDefaultHttp2(),
},
Upstream: &Upstream{
IdleTimeout: 10,
},
AddXRealIpHeader: true,
DisableXEnvoyHeaders: true,
},
@@ -194,6 +201,9 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) {
ConnectionBufferLimits: defaultConnectionBufferLimits,
Http2: NewDefaultHttp2(),
},
Upstream: &Upstream{
IdleTimeout: 10,
},
AddXRealIpHeader: true,
DisableXEnvoyHeaders: true,
},
@@ -206,6 +216,9 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) {
ConnectionBufferLimits: defaultConnectionBufferLimits,
Http2: NewDefaultHttp2(),
},
Upstream: &Upstream{
IdleTimeout: 10,
},
AddXRealIpHeader: true,
DisableXEnvoyHeaders: true,
},
@@ -214,6 +227,7 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) {
name: "delete and push",
old: &HigressConfig{
Downstream: NewDefaultDownstream(),
Upstream: NewDefaultUpStream(),
AddXRealIpHeader: defaultAddXRealIpHeader,
DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders,
},
@@ -222,6 +236,7 @@ func Test_AddOrUpdateHigressConfig(t *testing.T) {
wantEventPush: "push",
wantGlobal: &Global{
Downstream: NewDefaultDownstream(),
Upstream: NewDefaultUpStream(),
AddXRealIpHeader: defaultAddXRealIpHeader,
DisableXEnvoyHeaders: defaultDisableXEnvoyHeaders,
},

289
test/e2e/conformance/tests/configmap-global.go
View File

@@ -51,6 +51,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
@@ -117,6 +120,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
@@ -132,6 +145,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
DisableXEnvoyHeaders: true,
},
envoyAssertion: []envoy.Assertion{
@@ -190,6 +206,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
@@ -205,6 +231,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
@@ -268,6 +297,16 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
@@ -283,6 +322,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
},
envoyAssertion: []envoy.Assertion{
{
@@ -338,11 +380,24 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
name: "did not set Downstream, will use default value",
higressConfig: &configmap.HigressConfig{
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
@@ -405,6 +460,107 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
"idle_timeout": "180s",
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
name: "did not set Upstream, will use default value",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
@@ -486,10 +642,20 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
name: "did not set global config, downstream will use default value",
name: "did not set global config, downstream and upstream will use default value",
higressConfig: &configmap.HigressConfig{},
envoyAssertion: []envoy.Assertion{
{
@@ -541,10 +707,20 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
"idle_timeout": "180s",
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
name: "close the setting of idle timeout",
name: "close the setting of idle timeout in downstream",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 0,
@@ -556,6 +732,9 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 10,
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
@@ -618,6 +797,110 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
"idle_timeout": "0s",
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "10s",
},
},
},
},
},
{
name: "close the setting of idle timeout in upstream",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
Upstream: &configmap.Upstream{
IdleTimeout: 0,
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
{
Path: "configs.#.dynamic_active_clusters.#.cluster",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "0s",
},
},
},
},
},
}
@@ -629,7 +912,7 @@ var ConfigMapGlobalEnvoy = suite.ConformanceTest{
if err != nil {
t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress")
}
t.Logf("Checking Envoy config for test case %s", testcase.name)
t.Logf("Test Case %s", testcase.name)
for _, assertion := range testcase.envoyAssertion {
envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, assertion)
}