feat: add e2e test for envoy filter (#710)

This commit is contained in:
SJC
2023-12-29 21:39:28 +08:00
committed by GitHub
parent 85df257f4e
commit 89c72777e1
33 changed files with 1250 additions and 252 deletions

View File

@@ -0,0 +1,236 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"encoding/json"
"fmt"
"io"
"net/http"
"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
"github.com/alibaba/higress/pkg/cmd/options"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/yaml"
)
var (
BootstrapEnvoyConfigType EnvoyConfigType = "bootstrap"
ClusterEnvoyConfigType EnvoyConfigType = "cluster"
EndpointEnvoyConfigType EnvoyConfigType = "endpoint"
ListenerEnvoyConfigType EnvoyConfigType = "listener"
RouteEnvoyConfigType EnvoyConfigType = "route"
AllEnvoyConfigType EnvoyConfigType = "all"
)
const (
defaultProxyAdminPort = 15000
)
type EnvoyConfigType string
type GetEnvoyConfigOptions struct {
IncludeEds bool
PodName string
PodNamespace string
BindAddress string
Output string
EnvoyConfigType EnvoyConfigType
}
func NewDefaultGetEnvoyConfigOptions() *GetEnvoyConfigOptions {
return &GetEnvoyConfigOptions{
IncludeEds: true,
PodName: "",
PodNamespace: "higress-system",
BindAddress: "localhost",
Output: "json",
EnvoyConfigType: AllEnvoyConfigType,
}
}
func GetEnvoyConfig(config *GetEnvoyConfigOptions) ([]byte, error) {
configDump, err := retrieveConfigDump(config.PodName, config.PodNamespace, config.BindAddress, config.IncludeEds)
if err != nil {
return nil, err
}
if config.EnvoyConfigType == AllEnvoyConfigType {
return configDump, nil
}
resource, err := getXDSResource(config.EnvoyConfigType, configDump)
if err != nil {
return nil, err
}
return formatGatewayConfig(resource, config.Output)
}
func retrieveConfigDump(podName, podNamespace, bindAddress string, includeEds bool) ([]byte, error) {
if podNamespace == "" {
return nil, fmt.Errorf("pod namespace is required")
}
if podName == "" {
c, err := kubernetes.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader())
if err != nil {
return nil, fmt.Errorf("failed to build kubernetes client: %w", err)
}
podList, err := c.PodsForSelector(podNamespace, "app=higress-gateway")
if err != nil {
return nil, err
}
if len(podList.Items) == 0 {
return nil, fmt.Errorf("higress gateway pod is not existed in namespace %s", podNamespace)
}
podName = podList.Items[0].GetName()
}
fw, err := portForwarder(types.NamespacedName{
Namespace: podNamespace,
Name: podName,
}, bindAddress)
if err != nil {
return nil, err
}
if err := fw.Start(); err != nil {
return nil, err
}
defer fw.Stop()
configDump, err := fetchGatewayConfig(fw, includeEds)
if err != nil {
return nil, err
}
return configDump, nil
}
func portForwarder(nn types.NamespacedName, bindAddress string) (kubernetes.PortForwarder, error) {
c, err := kubernetes.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader())
if err != nil {
return nil, fmt.Errorf("build CLI client fail: %w", err)
}
pod, err := c.Pod(nn)
if err != nil {
return nil, fmt.Errorf("get pod %s fail: %w", nn, err)
}
if pod.Status.Phase != "Running" {
return nil, fmt.Errorf("pod %s is not running", nn)
}
fw, err := kubernetes.NewLocalPortForwarder(c, nn, 0, defaultProxyAdminPort, bindAddress)
if err != nil {
return nil, err
}
return fw, nil
}
func formatGatewayConfig(configDump any, output string) ([]byte, error) {
out, err := json.MarshalIndent(configDump, "", " ")
if err != nil {
return nil, err
}
if output == "yaml" {
out, err = yaml.JSONToYAML(out)
if err != nil {
return nil, err
}
}
return out, nil
}
func fetchGatewayConfig(fw kubernetes.PortForwarder, includeEds bool) ([]byte, error) {
out, err := configDumpRequest(fw.Address(), includeEds)
if err != nil {
return nil, err
}
return out, nil
}
func configDumpRequest(address string, includeEds bool) ([]byte, error) {
url := fmt.Sprintf("http://%s/config_dump", address)
if includeEds {
url = fmt.Sprintf("%s?include_eds", url)
}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
defer func() {
_ = resp.Body.Close()
}()
return io.ReadAll(resp.Body)
}
func getXDSResource(resourceType EnvoyConfigType, configDump []byte) (any, error) {
cd := map[string]any{}
if err := json.Unmarshal(configDump, &cd); err != nil {
return nil, err
}
if resourceType == AllEnvoyConfigType {
return cd, nil
}
configs := cd["configs"]
globalConfigs := configs.([]any)
switch resourceType {
case BootstrapEnvoyConfigType:
for _, config := range globalConfigs {
if config.(map[string]interface{})["@type"] == "type.googleapis.com/envoy.admin.v3.BootstrapConfigDump" {
return config, nil
}
}
case EndpointEnvoyConfigType:
for _, config := range globalConfigs {
if config.(map[string]interface{})["@type"] == "type.googleapis.com/envoy.admin.v3.EndpointsConfigDump" {
return config, nil
}
}
case ClusterEnvoyConfigType:
for _, config := range globalConfigs {
if config.(map[string]interface{})["@type"] == "type.googleapis.com/envoy.admin.v3.ClustersConfigDump" {
return config, nil
}
}
case ListenerEnvoyConfigType:
for _, config := range globalConfigs {
if config.(map[string]interface{})["@type"] == "type.googleapis.com/envoy.admin.v3.ListenersConfigDump" {
return config, nil
}
}
case RouteEnvoyConfigType:
for _, config := range globalConfigs {
if config.(map[string]interface{})["@type"] == "type.googleapis.com/envoy.admin.v3.RoutesConfigDump" {
return config, nil
}
}
default:
return nil, fmt.Errorf("unknown resourceType %s", resourceType)
}
return nil, fmt.Errorf("unknown resourceType %s", resourceType)
}

View File

@@ -0,0 +1,226 @@
// Copyright (c) 2022 Alibaba Group Holding Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"fmt"
"log"
"net"
"net/http"
"os"
"path"
"testing"
"github.com/alibaba/higress/pkg/cmd/hgctl/kubernetes"
"github.com/stretchr/testify/assert"
)
var _ kubernetes.PortForwarder = &fakePortForwarder{}
type fakePortForwarder struct {
responseBody []byte
localPort int
l net.Listener
mux *http.ServeMux
stopCh chan struct{}
}
func newFakePortForwarder(b []byte) (kubernetes.PortForwarder, error) {
p, err := kubernetes.LocalAvailablePort("localhost")
if err != nil {
return nil, err
}
fw := &fakePortForwarder{
responseBody: b,
localPort: p,
mux: http.NewServeMux(),
stopCh: make(chan struct{}),
}
fw.mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(fw.responseBody)
})
return fw, nil
}
func (fw *fakePortForwarder) WaitForStop() {
<-fw.stopCh
}
func (fw *fakePortForwarder) Start() error {
l, err := net.Listen("tcp", fw.Address())
if err != nil {
return err
}
fw.l = l
go func() {
if err := http.Serve(l, fw.mux); err != nil {
log.Fatal(err)
}
}()
return nil
}
func (fw *fakePortForwarder) Stop() {}
func (fw *fakePortForwarder) Address() string {
return fmt.Sprintf("localhost:%d", fw.localPort)
}
func TestExtractAllConfigDump(t *testing.T) {
input, err := readInputConfig("in.all.json")
assert.NoError(t, err)
fw, err := newFakePortForwarder(input)
assert.NoError(t, err)
err = fw.Start()
assert.NoError(t, err)
cases := []struct {
output string
expected string
resourceType string
}{
{
output: "json",
expected: "out.all.json",
},
{
output: "yaml",
expected: "out.all.yaml",
},
}
for _, tc := range cases {
t.Run(tc.output, func(t *testing.T) {
configDump, err := fetchGatewayConfig(fw, true)
assert.NoError(t, err)
data, err := getXDSResource(AllEnvoyConfigType, configDump)
assert.NoError(t, err)
got, err := formatGatewayConfig(data, tc.output)
assert.NoError(t, err)
out, err := readOutputConfig(tc.expected)
assert.NoError(t, err)
if tc.output == "yaml" {
assert.YAMLEq(t, string(out), string(got))
} else {
assert.JSONEq(t, string(out), string(got))
}
})
}
fw.Stop()
}
func TestExtractSubResourcesConfigDump(t *testing.T) {
input, err := readInputConfig("in.all.json")
assert.NoError(t, err)
fw, err := newFakePortForwarder(input)
assert.NoError(t, err)
err = fw.Start()
assert.NoError(t, err)
cases := []struct {
output string
expected string
resourceType EnvoyConfigType
}{
{
output: "json",
resourceType: BootstrapEnvoyConfigType,
expected: "out.bootstrap.json",
},
{
output: "yaml",
resourceType: BootstrapEnvoyConfigType,
expected: "out.bootstrap.yaml",
}, {
output: "json",
resourceType: ClusterEnvoyConfigType,
expected: "out.cluster.json",
},
{
output: "yaml",
resourceType: ClusterEnvoyConfigType,
expected: "out.cluster.yaml",
}, {
output: "json",
resourceType: ListenerEnvoyConfigType,
expected: "out.listener.json",
},
{
output: "yaml",
resourceType: ListenerEnvoyConfigType,
expected: "out.listener.yaml",
}, {
output: "json",
resourceType: RouteEnvoyConfigType,
expected: "out.route.json",
},
{
output: "yaml",
resourceType: RouteEnvoyConfigType,
expected: "out.route.yaml",
},
{
output: "json",
resourceType: EndpointEnvoyConfigType,
expected: "out.endpoints.json",
},
{
output: "yaml",
resourceType: EndpointEnvoyConfigType,
expected: "out.endpoints.yaml",
},
}
for _, tc := range cases {
t.Run(tc.output, func(t *testing.T) {
configDump, err := fetchGatewayConfig(fw, false)
assert.NoError(t, err)
resource, err := getXDSResource(tc.resourceType, configDump)
assert.NoError(t, err)
got, err := formatGatewayConfig(resource, tc.output)
assert.NoError(t, err)
out, err := readOutputConfig(tc.expected)
assert.NoError(t, err)
if tc.output == "yaml" {
assert.YAMLEq(t, string(out), string(got))
} else {
assert.JSONEq(t, string(out), string(got))
}
})
}
fw.Stop()
}
func readInputConfig(filename string) ([]byte, error) {
b, err := os.ReadFile(path.Join("testdata", "config", "input", filename))
if err != nil {
return nil, err
}
return b, nil
}
func readOutputConfig(filename string) ([]byte, error) {
b, err := os.ReadFile(path.Join("testdata", "config", "output", filename))
if err != nil {
return nil, err
}
return b, nil
}

View File

File diff suppressed because it is too large Load Diff

View File

File diff suppressed because it is too large Load Diff

View File

File diff suppressed because it is too large Load Diff

View File

File diff suppressed because it is too large Load Diff

View File

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,98 @@
{
"@type": "type.googleapis.com/envoy.admin.v3.ClustersConfigDump",
"version_info": "2",
"static_clusters": [{
"cluster": {
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "xds_cluster",
"type": "STRICT_DNS",
"connect_timeout": "1s",
"transport_socket": {
"name": "envoy.transport_sockets.tls",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"common_tls_context": {
"tls_params": {
"tls_maximum_protocol_version": "TLSv1_3"
},
"tls_certificate_sds_secret_configs": [{
"name": "xds_certificate",
"sds_config": {
"resource_api_version": "V3",
"path_config_source": {
"path": "/sds/xds-certificate.json"
}
}
}],
"validation_context_sds_secret_config": {
"name": "xds_trusted_ca",
"sds_config": {
"resource_api_version": "V3",
"path_config_source": {
"path": "/sds/xds-trusted-ca.json"
}
}
}
}
}
},
"load_assignment": {
"cluster_name": "xds_cluster",
"endpoints": [{
"lb_endpoints": [{
"endpoint": {
"address": {
"socket_address": {
"address": "higress",
"port_value": 18000
}
}
}
}]
}]
},
"typed_extension_protocol_options": {
"envoy.extensions.upstreams.http.v3.HttpProtocolOptions": {
"@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions",
"explicit_http_config": {
"http2_protocol_options": {}
}
}
}
},
"last_updated": "2023-02-23T09:05:23.436Z"
}],
"dynamic_active_clusters": [{
"version_info": "2a0a1698a9d3e05b802047b0cd36b52a070afa49042e1ba267168c5265c7cabf",
"cluster": {
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "default-backend-rule-0-match-0-www.example.com",
"type": "STATIC",
"connect_timeout": "5s",
"dns_lookup_family": "V4_ONLY",
"outlier_detection": {},
"common_lb_config": {
"locality_weighted_lb_config": {}
},
"load_assignment": {
"cluster_name": "default-backend-rule-0-match-0-www.example.com",
"endpoints": [{
"locality": {},
"lb_endpoints": [{
"endpoint": {
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 3000
}
}
},
"load_balancing_weight": 1
}],
"load_balancing_weight": 1
}]
}
},
"last_updated": "2023-02-23T09:05:38.443Z"
}]
}

View File

@@ -0,0 +1,67 @@
---
"@type": type.googleapis.com/envoy.admin.v3.ClustersConfigDump
version_info: '2'
static_clusters:
- cluster:
"@type": type.googleapis.com/envoy.config.cluster.v3.Cluster
name: xds_cluster
type: STRICT_DNS
connect_timeout: 1s
transport_socket:
name: envoy.transport_sockets.tls
typed_config:
"@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
common_tls_context:
tls_params:
tls_maximum_protocol_version: TLSv1_3
tls_certificate_sds_secret_configs:
- name: xds_certificate
sds_config:
resource_api_version: V3
path_config_source:
path: "/sds/xds-certificate.json"
validation_context_sds_secret_config:
name: xds_trusted_ca
sds_config:
resource_api_version: V3
path_config_source:
path: "/sds/xds-trusted-ca.json"
load_assignment:
cluster_name: xds_cluster
endpoints:
- lb_endpoints:
- endpoint:
address:
socket_address:
address: higress
port_value: 18000
typed_extension_protocol_options:
envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
"@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
explicit_http_config:
http2_protocol_options: {}
last_updated: '2023-02-23T09:05:23.436Z'
dynamic_active_clusters:
- version_info: 2a0a1698a9d3e05b802047b0cd36b52a070afa49042e1ba267168c5265c7cabf
cluster:
"@type": type.googleapis.com/envoy.config.cluster.v3.Cluster
name: default-backend-rule-0-match-0-www.example.com
type: STATIC
connect_timeout: 5s
dns_lookup_family: V4_ONLY
outlier_detection: {}
common_lb_config:
locality_weighted_lb_config: {}
load_assignment:
cluster_name: default-backend-rule-0-match-0-www.example.com
endpoints:
- locality: {}
lb_endpoints:
- endpoint:
address:
socket_address:
address: 0.0.0.0
port_value: 3000
load_balancing_weight: 1
load_balancing_weight: 1
last_updated: '2023-02-23T09:05:38.443Z'

View File

@@ -0,0 +1,30 @@
{
"@type": "type.googleapis.com/envoy.admin.v3.EndpointsConfigDump",
"staticEndpointConfigs": [{
"endpointConfig": {
"@type": "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment",
"clusterName": "xds_cluster",
"endpoints": [{
"locality": {},
"lbEndpoints": [{
"endpoint": {
"address": {
"socketAddress": {
"address": "0.0.0.0",
"portValue": 18000
}
},
"healthCheckConfig": {},
"hostname": "higress"
},
"healthStatus": "HEALTHY",
"metadata": {},
"loadBalancingWeight": 1
}]
}],
"policy": {
"overprovisioningFactor": 140
}
}
}]
}

View File

@@ -0,0 +1,21 @@
---
"@type": type.googleapis.com/envoy.admin.v3.EndpointsConfigDump
staticEndpointConfigs:
- endpointConfig:
"@type": type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment
clusterName: xds_cluster
endpoints:
- locality: {}
lbEndpoints:
- endpoint:
address:
socketAddress:
address: 0.0.0.0
portValue: 18000
healthCheckConfig: {}
hostname: higress
healthStatus: HEALTHY
metadata: {}
loadBalancingWeight: 1
policy:
overprovisioningFactor: 140

View File

@@ -0,0 +1,77 @@
{
"@type": "type.googleapis.com/envoy.admin.v3.ListenersConfigDump",
"version_info": "2",
"dynamic_listeners": [{
"name": "default-higress-http",
"active_state": {
"version_info": "42c71fb50c315ee3a32b327da69f8cc0baf420bc84b747e82d9c38e1b0c33eb2",
"listener": {
"@type": "type.googleapis.com/envoy.config.listener.v3.Listener",
"name": "default-higress-http",
"address": {
"socket_address": {
"address": "0.0.0.0",
"port_value": 10080
}
},
"access_log": [{
"name": "envoy.access_loggers.file",
"filter": {
"response_flag_filter": {
"flags": [
"NR"
]
}
},
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog",
"path": "/dev/stdout"
}
}],
"default_filter_chain": {
"filters": [{
"name": "envoy.filters.network.http_connection_manager",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager",
"stat_prefix": "http",
"rds": {
"config_source": {
"api_config_source": {
"api_type": "DELTA_GRPC",
"grpc_services": [{
"envoy_grpc": {
"cluster_name": "xds_cluster"
}
}],
"set_node_on_first_message_only": true,
"transport_api_version": "V3"
},
"resource_api_version": "V3"
},
"route_config_name": "default-higress-http"
},
"http_filters": [{
"name": "envoy.filters.http.router",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"
}
}],
"access_log": [{
"name": "envoy.access_loggers.file",
"typed_config": {
"@type": "type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog",
"path": "/dev/stdout"
}
}],
"use_remote_address": true,
"upgrade_configs": [{
"upgrade_type": "websocket"
}]
}
}]
}
},
"last_updated": "2023-02-23T09:05:38.446Z"
}
}]
}

View File

@@ -0,0 +1,53 @@
---
"@type": type.googleapis.com/envoy.admin.v3.ListenersConfigDump
version_info: '2'
dynamic_listeners:
- name: default-higress-http
active_state:
version_info: 42c71fb50c315ee3a32b327da69f8cc0baf420bc84b747e82d9c38e1b0c33eb2
listener:
"@type": type.googleapis.com/envoy.config.listener.v3.Listener
name: default-higress-http
address:
socket_address:
address: 0.0.0.0
port_value: 10080
access_log:
- name: envoy.access_loggers.file
filter:
response_flag_filter:
flags:
- NR
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: "/dev/stdout"
default_filter_chain:
filters:
- name: envoy.filters.network.http_connection_manager
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
stat_prefix: http
rds:
config_source:
api_config_source:
api_type: DELTA_GRPC
grpc_services:
- envoy_grpc:
cluster_name: xds_cluster
set_node_on_first_message_only: true
transport_api_version: V3
resource_api_version: V3
route_config_name: default-higress-http
http_filters:
- name: envoy.filters.http.router
typed_config:
"@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
access_log:
- name: envoy.access_loggers.file
typed_config:
"@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
path: "/dev/stdout"
use_remote_address: true
upgrade_configs:
- upgrade_type: websocket
last_updated: '2023-02-23T09:05:38.446Z'

View File

@@ -0,0 +1,31 @@
{
"@type": "type.googleapis.com/envoy.admin.v3.RoutesConfigDump",
"dynamic_route_configs": [{
"version_info": "cb1e51997a9c3aa6f4d920f39fd5bdbd966e9382b7b6bdf42efca8c22c6c3442",
"route_config": {
"@type": "type.googleapis.com/envoy.config.route.v3.RouteConfiguration",
"name": "default-higress-http",
"virtual_hosts": [{
"name": "default-higress-http",
"domains": [
"*"
],
"routes": [{
"match": {
"prefix": "/",
"headers": [{
"name": ":authority",
"string_match": {
"exact": "www.example.com"
}
}]
},
"route": {
"cluster": "default-backend-rule-0-match-0-www.example.com"
}
}]
}]
},
"last_updated": "2023-02-23T09:05:38.448Z"
}]
}

View File

@@ -0,0 +1,21 @@
---
"@type": type.googleapis.com/envoy.admin.v3.RoutesConfigDump
dynamic_route_configs:
- version_info: cb1e51997a9c3aa6f4d920f39fd5bdbd966e9382b7b6bdf42efca8c22c6c3442
route_config:
"@type": type.googleapis.com/envoy.config.route.v3.RouteConfiguration
name: default-higress-http
virtual_hosts:
- name: default-higress-http
domains:
- "*"
routes:
- match:
prefix: "/"
headers:
- name: ":authority"
string_match:
exact: www.example.com
route:
cluster: default-backend-rule-0-match-0-www.example.com
last_updated: '2023-02-23T09:05:38.448Z'