feat: add global-option setting in configmap (#699)

Signed-off-by: sjcsjc123 <1401189096@qq.com>
This commit is contained in:
SJC
2024-01-08 11:00:07 +08:00
committed by GitHub
parent 1f7e98cef5
commit c250e850d5
7 changed files with 1400 additions and 32 deletions

View File

@@ -0,0 +1,562 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package tests
import (
"testing"
"github.com/alibaba/higress/pkg/ingress/kube/configmap"
"github.com/alibaba/higress/test/e2e/conformance/utils/envoy"
"github.com/alibaba/higress/test/e2e/conformance/utils/kubernetes"
"github.com/alibaba/higress/test/e2e/conformance/utils/suite"
)
func init() {
Register(ConfigMapGlobalEnvoy)
}
var ConfigMapGlobalEnvoy = suite.ConformanceTest{
ShortName: "ConfigMapGlobalEnvoy",
Description: "The Envoy config should contain global config",
Manifests: []string{"tests/configmap-global.yaml"},
Features: []suite.SupportedFeature{suite.EnvoyConfigConformanceFeature},
Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
testCases := []struct {
name string
higressConfig *configmap.HigressConfig
envoyAssertion []envoy.Assertion
}{
{
name: "set config all",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
},
},
{
name: "did not set AddXRealIpHeader",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
DisableXEnvoyHeaders: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config.request_headers_to_add.#.header",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
},
},
{
name: "did not set DisableXEnvoyHeaders",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"typed_config": map[string]interface{}{
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
},
},
{
name: "did not set AddXRealIpHeader and DisableXEnvoyHeaders",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 180,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 100,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config.request_headers_to_add.#.header",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"typed_config": map[string]interface{}{
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "180s",
},
},
},
},
},
{
name: "did not set Downstream, will use default value",
higressConfig: &configmap.HigressConfig{
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"idle_timeout": "180s",
},
},
},
},
{
name: "modify Downstream",
higressConfig: &configmap.HigressConfig{
Downstream: &configmap.Downstream{
IdleTimeout: 200,
MaxRequestHeadersKb: 60,
ConnectionBufferLimits: 32768,
Http2: &configmap.Http2{
MaxConcurrentStreams: 200,
InitialStreamWindowSize: 65535,
InitialConnectionWindowSize: 1048576,
},
},
DisableXEnvoyHeaders: true,
AddXRealIpHeader: true,
},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"request_headers_to_add": []interface{}{
map[string]interface{}{
"append": false,
"header": map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeMatch,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"name": "envoy.filters.http.router",
"typed_config": map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router",
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"http2_protocol_options": map[string]interface{}{
"max_concurrent_streams": 200,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
},
"stream_idle_timeout": "200s",
"max_request_headers_kb": 60,
"common_http_protocol_options": map[string]interface{}{
"idle_timeout": "200s",
},
},
},
},
},
{
name: "did not set global config, downstream will use default value",
higressConfig: &configmap.HigressConfig{},
envoyAssertion: []envoy.Assertion{
{
Path: "configs.#.dynamic_route_configs.#.route_config.request_headers_to_add.#.header",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"key": "x-real-ip",
"value": "%REQ(X-ENVOY-EXTERNAL-ADDRESS)%",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "outbound_0.0.0.0_80",
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config.http_filters",
CheckType: envoy.CheckTypeNotExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"typed_config": map[string]interface{}{
"suppress_envoy_headers": true,
},
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"per_connection_buffer_limit_bytes": 32768,
},
},
{
Path: "configs.#.dynamic_listeners.#.active_state.listener.filter_chains.#.filters.#.typed_config",
CheckType: envoy.CheckTypeExist,
TargetNamespace: "higress-system",
ExpectEnvoyConfig: map[string]interface{}{
"max_concurrent_streams": 100,
"initial_stream_window_size": 65535,
"initial_connection_window_size": 1048576,
"stream_idle_timeout": "180s",
"max_request_headers_kb": 60,
"idle_timeout": "180s",
},
},
},
},
}
t.Run("ConfigMap Global Envoy", func(t *testing.T) {
for _, testcase := range testCases {
// apply config
err := kubernetes.ApplyConfigmapDataWithYaml(t, suite.Client, "higress-system", "higress-config", "higress", testcase.higressConfig)
if err != nil {
t.Fatalf("can't apply conifgmap %s in namespace %s for data key %s", "higress-config", "higress-system", "higress")
}
t.Logf("Checking Envoy config for test case %s", testcase.name)
for _, assertion := range testcase.envoyAssertion {
envoy.AssertEnvoyConfig(t, suite.TimeoutConfig, assertion)
}
}
})
},
}

View File

@@ -0,0 +1,32 @@
# Copyright (c) 2022 Alibaba Group Holding Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: higress-conformance-infra-configmap-global-test
namespace: higress-conformance-infra
spec:
ingressClassName: higress
rules:
- host: "foo.com"
http:
paths:
- pathType: Prefix
path: "/foo"
backend:
service:
name: infra-backend-v3
port:
number: 8080

View File

@@ -57,9 +57,42 @@ type Assertion struct {
func AssertEnvoyConfig(t *testing.T, timeoutConfig cfg.TimeoutConfig, expected Assertion) {
options := config.NewDefaultGetEnvoyConfigOptions()
options.PodNamespace = expected.TargetNamespace
convertEnvoyConfig := convertNumbersToFloat64(expected.ExpectEnvoyConfig)
if _, ok := convertEnvoyConfig.(map[string]interface{}); !ok {
t.Errorf("failed to convert envoy config number to float64")
return
}
expected.ExpectEnvoyConfig = convertEnvoyConfig.(map[string]interface{})
waitForEnvoyConfig(t, timeoutConfig, options, expected)
}
func convertNumbersToFloat64(data interface{}) interface{} {
switch val := data.(type) {
case map[string]interface{}:
result := make(map[string]interface{})
for key, v := range val {
result[key] = convertNumbersToFloat64(v)
}
return result
case []interface{}:
result := make([]interface{}, len(val))
for i, v := range val {
result[i] = convertNumbersToFloat64(v)
}
return result
case float64:
return val
case int:
return float64(val)
case int64:
return float64(val)
case float32:
return float64(val)
default:
return data
}
}
// waitForEnvoyConfig waits for the Envoy config to be ready and asserts it.
func waitForEnvoyConfig(t *testing.T, timeoutConfig cfg.TimeoutConfig, options *config.GetEnvoyConfigOptions, expected Assertion) {
awaitConvergence(t, defaultSuccessThreshold, timeoutConfig.MaxTimeToConsistency, func(elapsed time.Duration) bool {
@@ -245,7 +278,7 @@ func findKey(actual interface{}, key string, expectValue interface{}) bool {
case reflect.Map:
actualValueMap := actual.(map[string]interface{})
for actualKey, actualValue := range actualValueMap {
if actualKey == key && reflect.DeepEqual(convertType(actualValue, expectValue), expectValue) {
if actualKey == key && reflect.DeepEqual(actualValue, expectValue) {
return true
}
if findKey(actualValue, key, expectValue) {
@@ -254,34 +287,9 @@ func findKey(actual interface{}, key string, expectValue interface{}) bool {
}
return false
default:
if reflectValue.String() == key && reflect.DeepEqual(convertType(actual, expectValue), expectValue) {
if reflectValue.String() == key && reflect.DeepEqual(actual, expectValue) {
return true
}
return false
}
}
// convertType converts the type of the given value to the type of the given target value.
func convertType(value interface{}, targetType interface{}) interface{} {
targetTypeValue := reflect.ValueOf(targetType)
targetTypeKind := targetTypeValue.Kind()
switch targetTypeKind {
case reflect.Int:
switch value.(type) {
case int:
return value
case float64:
return int(value.(float64))
}
case reflect.Float64:
switch value.(type) {
case int:
return float64(value.(int))
case float64:
return value
}
}
return value
}