From 144d514eb0e30bfc15f7ef394463a7e69dd2f225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=BE=84=E6=BD=AD?= Date: Mon, 2 Sep 2024 22:02:25 +0800 Subject: [PATCH] Support new features: HTTP3/IPv6/ProxyProtocol/OpenTelemetry (#1272) --- Makefile.core.mk | 2 +- helm/core/templates/configmap.yaml | 6 +++++- helm/core/templates/controller-deployment.yaml | 4 ++++ helm/core/values.yaml | 3 +++ istio/istio | 2 +- pkg/ingress/kube/configmap/tracing.go | 14 +++++++------- pkg/ingress/kube/util/util.go | 7 ++++++- tools/hack/create-cluster.sh | 2 ++ tools/hack/prebuild.sh | 4 ++-- 9 files changed, 31 insertions(+), 13 deletions(-) diff --git a/Makefile.core.mk b/Makefile.core.mk index 7c1273d4e..891f81c19 100644 --- a/Makefile.core.mk +++ b/Makefile.core.mk @@ -188,7 +188,7 @@ install: pre-install helm install higress helm/higress -n higress-system --create-namespace --set 'global.local=true' ENVOY_LATEST_IMAGE_TAG ?= a6c313d41b3b54f0e3ed81fc676c520160cfed05 -ISTIO_LATEST_IMAGE_TAG ?= a6c313d41b3b54f0e3ed81fc676c520160cfed05 +ISTIO_LATEST_IMAGE_TAG ?= a9a55b3895bbf64a1ad8f724b2de3de017831e38 install-dev: pre-install helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true' diff --git a/helm/core/templates/configmap.yaml b/helm/core/templates/configmap.yaml index 02d0f3ad3..9a07c3392 100644 --- a/helm/core/templates/configmap.yaml +++ b/helm/core/templates/configmap.yaml @@ -9,7 +9,7 @@ accessLogFile: "/dev/stdout" {{- end }} ingressControllerMode: "OFF" - accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"} + accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%","response_code_details":"%RESPONSE_CODE_DETAILS%"} ' dnsRefreshRate: 200s @@ -32,6 +32,10 @@ - address: "k8s://" {{- end }} + mseIngressGlobalConfig: + enableH3: {{ .Values.global.enableH3 }} + enableProxyProtocol: {{ .Values.global.enableProxyProtocol }} + defaultConfig: {{- if .Values.global.disableAlpnH2 }} disableAlpnH2: true diff --git a/helm/core/templates/controller-deployment.yaml b/helm/core/templates/controller-deployment.yaml index 04912ebbc..494d099f8 100644 --- a/helm/core/templates/controller-deployment.yaml +++ b/helm/core/templates/controller-deployment.yaml @@ -131,6 +131,10 @@ spec: periodSeconds: 3 timeoutSeconds: 5 env: + - name: PILOT_ENABLE_QUIC_LISTENERS + value: "true" + - name: ISTIO_DUAL_STACK + value: "{{ .Values.global.enableIPv6 }}" - name: PILOT_ENABLE_HEADLESS_SERVICE_POD_LISTENERS value: "false" - name: PILOT_ENABLE_ALPN_FILTER diff --git a/helm/core/values.yaml b/helm/core/values.yaml index 2df6db87c..215e3cf2b 100644 --- a/helm/core/values.yaml +++ b/helm/core/values.yaml @@ -1,5 +1,8 @@ revision: "" global: + enableH3: false + enableIPv6: false + enableProxyProtocol: false liteMetrics: true xdsMaxRecvMsgSize: "104857600" defaultUpstreamConcurrencyThreshold: 10000 diff --git a/istio/istio b/istio/istio index 8a08c2421..c4fa0ca17 160000 --- a/istio/istio +++ b/istio/istio @@ -1 +1 @@ -Subproject commit 8a08c2421026122bdbdd8f95fb7a20a16e3b499f +Subproject commit c4fa0ca175754d8d97bc2af10471231b8442ec4f diff --git a/pkg/ingress/kube/configmap/tracing.go b/pkg/ingress/kube/configmap/tracing.go index 741676e17..209c610d8 100644 --- a/pkg/ingress/kube/configmap/tracing.go +++ b/pkg/ingress/kube/configmap/tracing.go @@ -334,7 +334,7 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s "@type": "type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig", "client_config": { "service_name": "higress-gateway.%s", - "backend_token": "%s" + "backend_token": "%s" }, "grpc_service": { "envoy_grpc": { @@ -363,11 +363,11 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s "name": "envoy.tracers.zipkin", "typed_config": { "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig", - "collector_cluster": "outbound|%s||%s", - "collector_endpoint": "/api/v2/spans", - "collector_hostname": "higress-gateway", - "collector_endpoint_version": "HTTP_JSON", - "split_spans_for_request": true + "collector_cluster": "outbound|%s||%s", + "collector_endpoint": "/api/v2/spans", + "collector_hostname": "higress-gateway", + "collector_endpoint_version": "HTTP_JSON", + "split_spans_for_request": true } }, "random_sampling": { @@ -389,7 +389,7 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s "name": "envoy.tracers.opentelemetry", "typed_config": { "@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig", - "service_name": "higress-gateway.%s" + "service_name": "higress-gateway.%s", "grpc_service": { "envoy_grpc": { "cluster_name": "outbound|%s||%s" diff --git a/pkg/ingress/kube/util/util.go b/pkg/ingress/kube/util/util.go index eaff16cb3..ec9c68870 100644 --- a/pkg/ingress/kube/util/util.go +++ b/pkg/ingress/kube/util/util.go @@ -29,6 +29,8 @@ import ( _struct "github.com/golang/protobuf/ptypes/struct" "istio.io/istio/pkg/cluster" "k8s.io/apimachinery/pkg/types" + + . "github.com/alibaba/higress/pkg/ingress/log" ) const ( @@ -105,6 +107,9 @@ func CreateServiceFQDN(namespace, name string) string { func BuildPatchStruct(config string) *_struct.Struct { val := &_struct.Struct{} - _ = jsonpb.Unmarshal(strings.NewReader(config), val) + err := jsonpb.Unmarshal(strings.NewReader(config), val) + if err != nil { + IngressLog.Errorf("build patch struct failed, err:%v", err) + } return val } diff --git a/tools/hack/create-cluster.sh b/tools/hack/create-cluster.sh index 854a523d4..d9e7907ca 100755 --- a/tools/hack/create-cluster.sh +++ b/tools/hack/create-cluster.sh @@ -43,6 +43,8 @@ cat < "tools/hack/cluster.conf" # cluster.conf kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +networking: + ipFamily: dual nodes: - role: control-plane kubeadmConfigPatches: diff --git a/tools/hack/prebuild.sh b/tools/hack/prebuild.sh index 25b04eb77..944091627 100755 --- a/tools/hack/prebuild.sh +++ b/tools/hack/prebuild.sh @@ -15,7 +15,7 @@ for repo in ${envoy_repos[@]}; do if [ -e external/$repo ];then continue fi - cp -rP envoy/$repo external/$repo + cp -RP envoy/$repo external/$repo cd external/$repo echo "gitdir: /parent/.git/modules/envoy/$repo" > .git if [ -f "go.mod" ]; then @@ -30,7 +30,7 @@ for repo in ${istio_repos[@]}; do if [ -e external/$repo ];then continue fi - cp -rP istio/$repo external/$repo + cp -RP istio/$repo external/$repo cd external/$repo echo "gitdir: /parent/.git/modules/istio/$repo" > .git if [ -f "go.mod" ]; then