feat: add annotation for mirror svc (#1121)

pkg/ingress/kube/annotations/annotations.go

@@ -69,6 +69,8 @@ type Ingress struct {
 
 	Auth *AuthConfig
 
+	Mirror *MirrorConfig
+
 	Destination *DestinationConfig
 
 	IgnoreCase *IgnoreCaseConfig
@@ -161,6 +163,7 @@ func NewAnnotationHandlerManager() AnnotationHandler {
 			localRateLimit{},
 			fallback{},
 			auth{},
+			mirror{},
 			destination{},
 			ignoreCaseMatching{},
 			match{},
@@ -182,6 +185,7 @@ func NewAnnotationHandlerManager() AnnotationHandler {
 			retry{},
 			localRateLimit{},
 			fallback{},
+			mirror{},
 			ignoreCaseMatching{},
 			match{},
 			headerControl{},

pkg/ingress/kube/annotations/mirror.go

@@ -0,0 +1,118 @@
+// Copyright (c) 2023 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package annotations
+
+import (
+	"github.com/alibaba/higress/pkg/ingress/kube/util"
+	. "github.com/alibaba/higress/pkg/ingress/log"
+	wrappers "google.golang.org/protobuf/types/known/wrapperspb"
+	networking "istio.io/api/networking/v1alpha3"
+)
+
+const (
+	mirrorTargetService = "mirror-target-service"
+	mirrorPercentage    = "mirror-percentage"
+)
+
+var (
+	_ Parser       = &mirror{}
+	_ RouteHandler = &mirror{}
+)
+
+type MirrorConfig struct {
+	util.ServiceInfo
+	Percentage *wrappers.DoubleValue
+}
+
+type mirror struct{}
+
+func (m mirror) Parse(annotations Annotations, config *Ingress, globalContext *GlobalContext) error {
+	if !needMirror(annotations) {
+		return nil
+	}
+
+	target, err := annotations.ParseStringASAP(mirrorTargetService)
+	if err != nil {
+		IngressLog.Errorf("Get mirror target service fail, err: %v", err)
+		return nil
+	}
+
+	serviceInfo, err := util.ParseServiceInfo(target, config.Namespace)
+	if err != nil {
+		IngressLog.Errorf("Get mirror target service fail, err: %v", err)
+		return nil
+	}
+
+	serviceLister, exist := globalContext.ClusterServiceList[config.ClusterId]
+	if !exist {
+		IngressLog.Errorf("service lister of cluster %s doesn't exist", config.ClusterId)
+		return nil
+	}
+
+	service, err := serviceLister.Services(serviceInfo.Namespace).Get(serviceInfo.Name)
+	if err != nil {
+		IngressLog.Errorf("Mirror service %s/%s within ingress %s/%s is not found, with err: %v",
+			serviceInfo.Namespace, serviceInfo.Name, config.Namespace, config.Name, err)
+		return nil
+	}
+	if service == nil {
+		IngressLog.Errorf("service %s/%s within ingress %s/%s is empty value",
+			serviceInfo.Namespace, serviceInfo.Name, config.Namespace, config.Name)
+		return nil
+	}
+
+	if serviceInfo.Port == 0 {
+		// Use the first port
+		serviceInfo.Port = uint32(service.Spec.Ports[0].Port)
+	}
+
+	var percentage *wrappers.DoubleValue
+	if value, err := annotations.ParseIntASAP(mirrorPercentage); err == nil {
+		if value < 100 {
+			percentage = &wrappers.DoubleValue{
+				Value: float64(value),
+			}
+		}
+	}
+
+	config.Mirror = &MirrorConfig{
+		ServiceInfo: serviceInfo,
+		Percentage:  percentage,
+	}
+	return nil
+}
+
+func (m mirror) ApplyRoute(route *networking.HTTPRoute, config *Ingress) {
+	if config.Mirror == nil {
+		return
+	}
+
+	route.Mirror = &networking.Destination{
+		Host: util.CreateServiceFQDN(config.Mirror.Namespace, config.Mirror.Name),
+		Port: &networking.PortSelector{
+			Number: config.Mirror.Port,
+		},
+	}
+
+	if config.Mirror.Percentage != nil {
+		route.MirrorPercentage = &networking.Percent{
+			Value: config.Mirror.Percentage.GetValue(),
+		}
+	}
+}
+
+func needMirror(annotations Annotations) bool {
+	return annotations.HasASAP(mirrorTargetService)
+}

pkg/ingress/kube/annotations/mirror_test.go

@@ -0,0 +1,163 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package annotations
+
+import (
+	"github.com/alibaba/higress/pkg/ingress/kube/util"
+	"github.com/golang/protobuf/proto"
+	networking "istio.io/api/networking/v1alpha3"
+	"istio.io/istio/pilot/pkg/model"
+	"reflect"
+	"testing"
+)
+
+func TestParseMirror(t *testing.T) {
+	testCases := []struct {
+		input  []map[string]string
+		expect *MirrorConfig
+	}{
+		{},
+		{
+			input: []map[string]string{
+				{buildHigressAnnotationKey(mirrorTargetService): "test/app"},
+				{buildNginxAnnotationKey(mirrorTargetService): "test/app"},
+			},
+			expect: &MirrorConfig{
+				ServiceInfo: util.ServiceInfo{
+					NamespacedName: model.NamespacedName{
+						Namespace: "test",
+						Name:      "app",
+					},
+					Port: 80,
+				},
+			},
+		},
+		{
+			input: []map[string]string{
+				{buildHigressAnnotationKey(mirrorTargetService): "test/app:8080"},
+				{buildNginxAnnotationKey(mirrorTargetService): "test/app:8080"},
+			},
+			expect: &MirrorConfig{
+				ServiceInfo: util.ServiceInfo{
+					NamespacedName: model.NamespacedName{
+						Namespace: "test",
+						Name:      "app",
+					},
+					Port: 8080,
+				},
+			},
+		},
+		{
+			input: []map[string]string{
+				{buildHigressAnnotationKey(mirrorTargetService): "test/app:hi"},
+				{buildNginxAnnotationKey(mirrorTargetService): "test/app:hi"},
+			},
+			expect: &MirrorConfig{
+				ServiceInfo: util.ServiceInfo{
+					NamespacedName: model.NamespacedName{
+						Namespace: "test",
+						Name:      "app",
+					},
+					Port: 80,
+				},
+			},
+		},
+		{
+			input: []map[string]string{
+				{buildHigressAnnotationKey(mirrorTargetService): "test/app"},
+				{buildNginxAnnotationKey(mirrorTargetService): "test/app"},
+			},
+			expect: &MirrorConfig{
+				ServiceInfo: util.ServiceInfo{
+					NamespacedName: model.NamespacedName{
+						Namespace: "test",
+						Name:      "app",
+					},
+					Port: 80,
+				},
+			},
+		},
+	}
+
+	mirror := mirror{}
+
+	for _, testCase := range testCases {
+		t.Run("", func(t *testing.T) {
+			config := &Ingress{
+				Meta: Meta{
+					Namespace: "test",
+					ClusterId: "cluster",
+				},
+			}
+			globalContext, cancel := initGlobalContextForService()
+			defer cancel()
+
+			for _, in := range testCase.input {
+				_ = mirror.Parse(in, config, globalContext)
+				if !reflect.DeepEqual(testCase.expect, config.Mirror) {
+					t.Log("expect:", *testCase.expect)
+					t.Log("actual:", *config.Mirror)
+					t.Fatal("Should be equal")
+				}
+			}
+		})
+	}
+}
+
+func TestMirror_ApplyRoute(t *testing.T) {
+	testCases := []struct {
+		config *Ingress
+		input  *networking.HTTPRoute
+		expect *networking.HTTPRoute
+	}{
+		{
+			config: &Ingress{},
+			input:  &networking.HTTPRoute{},
+			expect: &networking.HTTPRoute{},
+		},
+		{
+			config: &Ingress{
+				Mirror: &MirrorConfig{
+					ServiceInfo: util.ServiceInfo{
+						NamespacedName: model.NamespacedName{
+							Namespace: "default",
+							Name:      "test",
+						},
+						Port: 8080,
+					},
+				},
+			},
+			input: &networking.HTTPRoute{},
+			expect: &networking.HTTPRoute{
+				Mirror: &networking.Destination{
+					Host: "test.default.svc.cluster.local",
+					Port: &networking.PortSelector{
+						Number: 8080,
+					},
+				},
+			},
+		},
+	}
+
+	mirror := mirror{}
+	for _, testCase := range testCases {
+		t.Run("", func(t *testing.T) {
+			mirror.ApplyRoute(testCase.input, testCase.config)
+			if !proto.Equal(testCase.input, testCase.expect) {
+				t.Fatal("Must be equal.")
+			}
+		})
+	}
+}

pkg/ingress/kube/util/util.go

@@ -20,8 +20,10 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"istio.io/istio/pilot/pkg/model"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/golang/protobuf/jsonpb"
@@ -113,3 +115,44 @@ func BuildPatchStruct(config string) *_struct.Struct {
 	}
 	return val
 }
+
+type ServiceInfo struct {
+	model.NamespacedName
+	Port uint32
+}
+
+// convertToPort converts a port string to a uint32.
+func convertToPort(v string) (uint32, error) {
+	p, err := strconv.ParseUint(v, 10, 32)
+	if err != nil || p > 65535 {
+		return 0, fmt.Errorf("invalid port %s: %v", v, err)
+	}
+	return uint32(p), nil
+}
+
+func ParseServiceInfo(service string, ingressNamespace string) (ServiceInfo, error) {
+	parts := strings.Split(service, ":")
+	namespacedName := SplitNamespacedName(parts[0])
+
+	if namespacedName.Name == "" {
+		return ServiceInfo{}, errors.New("service name can not be empty")
+	}
+
+	if namespacedName.Namespace == "" {
+		namespacedName.Namespace = ingressNamespace
+	}
+
+	var port uint32
+	if len(parts) == 2 {
+		// If port parse fail, we ignore port and pick the first one.
+		port, _ = convertToPort(parts[1])
+	}
+
+	return ServiceInfo{
+		NamespacedName: model.NamespacedName{
+			Name:      namespacedName.Name,
+			Namespace: namespacedName.Namespace,
+		},
+		Port: port,
+	}, nil
+}

plugins/wasm-go/extensions/oidc/go.mod

@@ -1,6 +1,8 @@
 module github.com/alibaba/higress/plugins/wasm-go/extensions/oidc
 
-go 1.19
+go 1.21
+
+toolchain go1.22.5
 
 replace github.com/alibaba/higress/plugins/wasm-go => ../..
 

test/e2e/conformance/base/manifests.yaml

@@ -178,6 +178,55 @@ spec:
 ---
 apiVersion: v1
 kind: Service
+metadata:
+  name: infra-backend-mirror
+  namespace: higress-conformance-infra
+spec:
+  selector:
+    app: infra-backend-mirror
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 3000
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: infra-backend-mirror
+  namespace: higress-conformance-infra
+  labels:
+    app: infra-backend-mirror
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: infra-backend-mirror
+  template:
+    metadata:
+      labels:
+        app: infra-backend-mirror
+    spec:
+      containers:
+        - name: infra-backend-mirror
+          # image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echoserver:v20221109-7ee2f3e
+
+          # From https://github.com/Uncle-Justice/echo-server
+          image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/echo-server:1.3.0
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          resources:
+            requests:
+              cpu: 10m
+---
+apiVersion: v1
+kind: Service
 metadata:
   name: infra-backend-echo-body-v1
   namespace: higress-conformance-infra

test/e2e/conformance/tests/httproute-mirror-target-service.go

@@ -0,0 +1,107 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tests
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/alibaba/higress/test/e2e/conformance/utils/http"
+	"github.com/alibaba/higress/test/e2e/conformance/utils/suite"
+	v1 "k8s.io/api/core/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+func init() {
+	Register(HTTPRouteMirrorTargetService)
+}
+
+var HTTPRouteMirrorTargetService = suite.ConformanceTest{
+	ShortName:   "HTTPRouteMirrorTargetService",
+	Description: "The Ingress in the higress-conformance-infra namespace mirror request to target service",
+	Features:    []suite.SupportedFeature{suite.HTTPConformanceFeature},
+	Manifests:   []string{"tests/httproute-mirror-target-service.yaml"},
+	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
+		testcases := []http.Assertion{
+			{
+				Meta: http.AssertionMeta{
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Path: "/mirror",
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 200,
+					},
+				},
+			},
+		}
+
+		t.Run("HTTPRoute mirror request to target service", func(t *testing.T) {
+			for _, testcase := range testcases {
+				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase)
+				//check mirror's logs for request
+				cfg, err := config.GetConfig()
+				if err != nil {
+					t.Fatalf("[httproute-mirror] get config failed.")
+					return
+				}
+				clientSet, err := kubernetes.NewForConfig(cfg)
+				if err != nil {
+					t.Fatalf("[httproute-mirror] init clientset failed.")
+					return
+				}
+				pods, err := clientSet.CoreV1().Pods("higress-conformance-infra").List(context.Background(), meta_v1.ListOptions{
+					LabelSelector: meta_v1.FormatLabelSelector(&meta_v1.LabelSelector{MatchLabels: map[string]string{"app": "infra-backend-mirror"}}),
+				})
+				if err != nil || len(pods.Items) == 0 {
+					t.Fatalf("[httproute-mirror] get pods by label of [\"app\": \"infra-backend-mirror\"] failed.")
+					return
+				}
+				req := clientSet.CoreV1().Pods("higress-conformance-infra").GetLogs(pods.Items[0].Name, &v1.PodLogOptions{
+					Container: "infra-backend-mirror",
+					SinceTime: &meta_v1.Time{Time: time.Now().Add(-time.Second * 10)},
+				})
+				podLogs, err := req.Stream(context.Background())
+				defer podLogs.Close()
+				if err != nil {
+					t.Fatalf("[httproute-mirror] init pod logs stream failed.")
+					return
+				}
+
+				podBuf := new(bytes.Buffer)
+				_, err = io.Copy(podBuf, podLogs)
+				if err != nil {
+					t.Fatalf("[httproute-mirror] read pod logs stream failed.")
+					return
+				}
+				if !strings.Contains(podBuf.String(), "Echoing back request made to /mirror") {
+					t.Fatalf("[httproute-mirror] mirror pod hasn't received any mirror requests in logs.")
+					return
+				}
+			}
+		})
+	},
+}

test/e2e/conformance/tests/httproute-mirror-target-service.yaml

@@ -0,0 +1,32 @@
+# Copyright (c) 2022 Alibaba Group Holding Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: higress-conformance-infra-mirror-target-service
+  namespace: higress-conformance-infra
+  annotations:
+    nginx.ingress.kubernetes.io/mirror-target-service: "infra-backend-mirror"
+spec:
+  ingressClassName: higress
+  rules:
+  - http:
+      paths:
+      - pathType: Prefix
+        path: "/mirror"
+        backend:
+          service:
+            name: infra-backend-v1
+            port:
+              number: 8080