Support new features: HTTP3/IPv6/ProxyProtocol/OpenTelemetry (#1272)

Makefile.core.mk

@@ -188,7 +188,7 @@ install: pre-install
 	helm install higress helm/higress -n higress-system --create-namespace --set 'global.local=true'
 
 ENVOY_LATEST_IMAGE_TAG ?= a6c313d41b3b54f0e3ed81fc676c520160cfed05
-ISTIO_LATEST_IMAGE_TAG ?= a6c313d41b3b54f0e3ed81fc676c520160cfed05
+ISTIO_LATEST_IMAGE_TAG ?= a9a55b3895bbf64a1ad8f724b2de3de017831e38
 
 install-dev: pre-install
 	helm install higress helm/core -n higress-system --create-namespace --set 'controller.tag=$(TAG)' --set 'gateway.replicas=1' --set 'pilot.tag=$(ISTIO_LATEST_IMAGE_TAG)' --set 'gateway.tag=$(ENVOY_LATEST_IMAGE_TAG)' --set 'global.local=true'

helm/core/templates/configmap.yaml

@@ -9,7 +9,7 @@
     accessLogFile: "/dev/stdout"
     {{- end }}
     ingressControllerMode: "OFF"
-    accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%"}
+    accessLogFormat: '{"authority":"%REQ(X-ENVOY-ORIGINAL-HOST?:AUTHORITY)%","bytes_received":"%BYTES_RECEIVED%","bytes_sent":"%BYTES_SENT%","downstream_local_address":"%DOWNSTREAM_LOCAL_ADDRESS%","downstream_remote_address":"%DOWNSTREAM_REMOTE_ADDRESS%","duration":"%DURATION%","istio_policy_status":"%DYNAMIC_METADATA(istio.mixer:status)%","method":"%REQ(:METHOD)%","path":"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%","protocol":"%PROTOCOL%","request_id":"%REQ(X-REQUEST-ID)%","requested_server_name":"%REQUESTED_SERVER_NAME%","response_code":"%RESPONSE_CODE%","response_flags":"%RESPONSE_FLAGS%","route_name":"%ROUTE_NAME%","start_time":"%START_TIME%","trace_id":"%REQ(X-B3-TRACEID)%","upstream_cluster":"%UPSTREAM_CLUSTER%","upstream_host":"%UPSTREAM_HOST%","upstream_local_address":"%UPSTREAM_LOCAL_ADDRESS%","upstream_service_time":"%RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%","upstream_transport_failure_reason":"%UPSTREAM_TRANSPORT_FAILURE_REASON%","user_agent":"%REQ(USER-AGENT)%","x_forwarded_for":"%REQ(X-FORWARDED-FOR)%","response_code_details":"%RESPONSE_CODE_DETAILS%"}
 
     '
     dnsRefreshRate: 200s
@@ -32,6 +32,10 @@
       - address: "k8s://"
     {{- end }}
 
+    mseIngressGlobalConfig:
+      enableH3: {{ .Values.global.enableH3 }}
+      enableProxyProtocol: {{ .Values.global.enableProxyProtocol }}
+
     defaultConfig:
       {{- if .Values.global.disableAlpnH2 }}
       disableAlpnH2: true

helm/core/templates/controller-deployment.yaml

@@ -131,6 +131,10 @@ spec:
             periodSeconds: 3
             timeoutSeconds: 5
           env:
+          - name: PILOT_ENABLE_QUIC_LISTENERS
+            value: "true"
+          - name: ISTIO_DUAL_STACK
+            value: "{{ .Values.global.enableIPv6 }}"
           - name: PILOT_ENABLE_HEADLESS_SERVICE_POD_LISTENERS
             value: "false"
           - name: PILOT_ENABLE_ALPN_FILTER

helm/core/values.yaml

@@ -1,5 +1,8 @@
 revision: ""
 global:
+  enableH3: false
+  enableIPv6: false
+  enableProxyProtocol: false
   liteMetrics: true
   xdsMaxRecvMsgSize: "104857600"
   defaultUpstreamConcurrencyThreshold: 10000

pkg/ingress/kube/configmap/tracing.go

@@ -334,7 +334,7 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s
 					"@type": "type.googleapis.com/envoy.config.trace.v3.SkyWalkingConfig",
 					"client_config": {
 						"service_name": "higress-gateway.%s",
-                        "backend_token": "%s"
+                                                "backend_token": "%s"
 					},
 					"grpc_service": {
 						"envoy_grpc": {
@@ -363,11 +363,11 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s
 				"name": "envoy.tracers.zipkin",
 				"typed_config": {
 					"@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
-                    "collector_cluster": "outbound|%s||%s",
+                                        "collector_cluster": "outbound|%s||%s",
-                    "collector_endpoint": "/api/v2/spans",
+                                        "collector_endpoint": "/api/v2/spans",
-                    "collector_hostname": "higress-gateway",
+                                        "collector_hostname": "higress-gateway",
-                    "collector_endpoint_version": "HTTP_JSON",
+                                        "collector_endpoint_version": "HTTP_JSON",
-                    "split_spans_for_request": true
+                                        "split_spans_for_request": true
 				}
 			},
 			"random_sampling": {
@@ -389,7 +389,7 @@ func (t *TracingController) constructTracingTracer(tracing *Tracing, namespace s
 				"name": "envoy.tracers.opentelemetry",
 				"typed_config": {
 					"@type": "type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig",
-					"service_name": "higress-gateway.%s"
+					"service_name": "higress-gateway.%s",
 					"grpc_service": {
 						"envoy_grpc": {
 							"cluster_name": "outbound|%s||%s"

pkg/ingress/kube/util/util.go

@@ -29,6 +29,8 @@ import (
 	_struct "github.com/golang/protobuf/ptypes/struct"
 	"istio.io/istio/pkg/cluster"
 	"k8s.io/apimachinery/pkg/types"
+
+	. "github.com/alibaba/higress/pkg/ingress/log"
 )
 
 const (
@@ -105,6 +107,9 @@ func CreateServiceFQDN(namespace, name string) string {
 
 func BuildPatchStruct(config string) *_struct.Struct {
 	val := &_struct.Struct{}
-	_ = jsonpb.Unmarshal(strings.NewReader(config), val)
+	err := jsonpb.Unmarshal(strings.NewReader(config), val)
+	if err != nil {
+		IngressLog.Errorf("build patch struct failed, err:%v", err)
+	}
 	return val
 }

tools/hack/create-cluster.sh

@@ -43,6 +43,8 @@ cat <<EOF > "tools/hack/cluster.conf"
 # cluster.conf
 kind: Cluster
 apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  ipFamily: dual
 nodes:
 - role: control-plane
   kubeadmConfigPatches:

tools/hack/prebuild.sh

@@ -15,7 +15,7 @@ for repo in ${envoy_repos[@]}; do
     if [ -e external/$repo ];then
         continue
     fi
-    cp -rP envoy/$repo  external/$repo
+    cp -RP envoy/$repo  external/$repo
     cd external/$repo
     echo "gitdir: /parent/.git/modules/envoy/$repo" > .git
     if [ -f "go.mod" ]; then
@@ -30,7 +30,7 @@ for repo in ${istio_repos[@]}; do
     if [ -e external/$repo ];then
         continue
     fi
-    cp -rP istio/$repo external/$repo
+    cp -RP istio/$repo external/$repo
     cd external/$repo
     echo "gitdir: /parent/.git/modules/istio/$repo" > .git
     if [ -f "go.mod" ]; then