feat(transformer): Add split and retain strategy for dedupe (#2761)

plugins/wasm-go/extensions/transformer/README.md

@@ -74,7 +74,7 @@ description: 请求响应转换插件配置参考
 | 添加 add      | 添加的 key | 添加的 value | 若不存在指定的 `key:value`，则添加；否则无操作                                                                                                                                                                                                                                                         |
 | 追加 append   | 目标 key |追加的 value值 appendValue| 若存在指定的 `key:value`，则追加 appendValue 得到 `key:[value..., appendValue]`；否则相当于执行 add 操作，得到 `key:appendValue`                                                                                                                                                                               |
 | 映射 map      | 映射来源 fromKey |映射目标 toKey| 若存在指定的 `fromKey:fromValue`，则将其值 fromValue 映射给 toKey 的值，得到 `toKey:fromValue`，同时保留 `fromKey:fromValue`（注：若 toKey 已存在则其值会被覆盖）；否则无操作                                                                                                                                                      |
-| 去重 dedupe   | 目标 key |指定去重策略 strategy| `strategy` 可选值为：<br>`RETAIN_UNIQUE`: 按顺序保留所有唯一值，如 `k1:[v1,v2,v3,v3,v2,v1]`，去重后得到 `k1:[v1,v2,v3]` <br>`RETAIN_LAST`: 保留最后一个值，如 `k1:[v1,v2,v3]`，去重后得到 `k1:v3` <br>`RETAIN_FIRST` (default): 保留第一个值，如 `k1:[v1,v2,v3]`，去重后得到 `k1:v1`<br>（注：若去重后只剩下一个元素 v1 时，键值对变为 `k1:v1`, 而不是 `k1:[v1]`） |
+| 去重 dedupe   | 目标 key |指定去重策略 strategy| `strategy` 可选值为：<br>`RETAIN_UNIQUE`: 按顺序保留所有唯一值，如 `k1:[v1,v2,v3,v3,v2,v1]`，去重后得到 `k1:[v1,v2,v3]` <br>`RETAIN_LAST`: 保留最后一个值，如 `k1:[v1,v2,v3]`，去重后得到 `k1:v3` <br>`RETAIN_FIRST` (default): 保留第一个值，如 `k1:[v1,v2,v3]`，去重后得到 `k1:v1`<br>`SPLIT_AND_RETAIN_FIRST`: 对值按逗号进行切割，并保留第一个值，如 `k1:"v1,v2,v3"`，去重后得到 `k1:v1`<br>`SPLIT_AND_RETAIN_LAST`: 对值按逗号进行切割，并保留最后一个值，如 `k1:"v1,v2,v3"`，去重后得到 `k1:v3`<br>（注：若去重后只剩下一个元素 v1 时，键值对变为 `k1:v1`, 而不是 `k1:[v1]`） |
 
 
 

plugins/wasm-go/extensions/transformer/README_EN.md

@@ -66,7 +66,7 @@ Note:
 | Add add | Added key | Added value | If the specified `key:value` does not exist, add it; otherwise, no operation |
 | Append append | Target key | Appending value appendValue | If the specified `key:value` exists, append appendValue to get `key:[value..., appendValue]`; otherwise, it is equivalent to performing add operation, resulting in `key:appendValue`. |
 | Map map | Mapping source fromKey | Mapping target toKey | If the specified `fromKey:fromValue` exists, map its value fromValue to the value of toKey, resulting in `toKey:fromValue`, while retaining `fromKey:fromValue` (note: if toKey already exists, its value will be overwritten); otherwise, no operation. |
-| Deduplicate dedupe | Target key | Specified deduplication strategy strategy | `strategy` optional values include: <br>`RETAIN_UNIQUE`: Retain all unique values in order, e.g., `k1:[v1,v2,v3,v3,v2,v1]`, deduplication results in `k1:[v1,v2,v3]`. <br>`RETAIN_LAST`: Retain the last value, e.g., `k1:[v1,v2,v3]`, deduplication results in `k1:v3`. <br>`RETAIN_FIRST` (default): Retain the first value, e.g., `k1:[v1,v2,v3]`, deduplication results in `k1:v1`. <br>(Note: When deduplication results in only one element v1, the key-value pair becomes `k1:v1`, not `k1:[v1]`.) |
+| Deduplicate dedupe | Target key | Specified deduplication strategy strategy | `strategy` optional values include: <br>`RETAIN_UNIQUE`: Retain all unique values in order, e.g., `k1:[v1,v2,v3,v3,v2,v1]`, deduplication results in `k1:[v1,v2,v3]`. <br>`RETAIN_LAST`: Retain the last value, e.g., `k1:[v1,v2,v3]`, deduplication results in `k1:v3`. <br>`RETAIN_FIRST` (default): Retain the first value, e.g., `k1:[v1,v2,v3]`, deduplication results in `k1:v1`. <br>`SPLIT_AND_RETAIN_FIRST`: Split the value by comma and retain the first value, e.g., `k1:"v1,v2,v3"`, deduplication results in `k1:v1`. <br>`SPLIT_AND_RETAIN_LAST`: Split the value by comma and retain the last value, e.g., `k1:"v1,v2,v3"`, deduplication results in `k1:v3`. <br>(Note: When deduplication results in only one element v1, the key-value pair becomes `k1:v1`, not `k1:[v1]`.) |
 
 ## Configuration Example
 

plugins/wasm-go/extensions/transformer/main.go

@@ -1011,7 +1011,15 @@ func (h kvHandler) handle(host, path string, kvs map[string][]string, mapSourceD
 					if vs, ok := kvs[key]; ok && len(vs) >= 1 {
 						kvs[key] = vs[len(vs)-1:]
 					}
-
+				case "SPLIT_AND_RETAIN_FIRST":
+					if vs, ok := kvs[key]; ok && len(vs) >= 1 {
+						kvs[key] = strings.Split(vs[0], ",")[:1]
+					}
+				case "SPLIT_AND_RETAIN_LAST":
+					if vs, ok := kvs[key]; ok && len(vs) >= 1 {
+						split := strings.Split(vs[0], ",")
+						kvs[key] = split[len(split)-1:]
+					}
 				case "RETAIN_FIRST":
 					fallthrough
 				default:
@@ -1202,7 +1210,20 @@ func (h jsonHandler) handle(host, path string, oriData []byte, mapSourceData map
 
 				case "RETAIN_LAST":
 					dedupedVal = values[len(values)-1].Value() // key: last
-
+				case "SPLIT_AND_RETAIN_FIRST":
+					if len(values) > 0 {
+						split := strings.Split(values[0].String(), ",")
+						if len(split) > 0 {
+							dedupedVal = split[0]
+						}
+					}
+				case "SPLIT_AND_RETAIN_LAST":
+					if len(values) > 0 {
+						split := strings.Split(values[0].String(), ",")
+						if len(split) > 0 {
+							dedupedVal = split[len(split)-1]
+						}
+					}
 				case "RETAIN_FIRST":
 					fallthrough
 				default:

test/e2e/conformance/tests/go-wasm-transformer.go

@@ -632,6 +632,38 @@ var WasmPluginsTransformer = suite.ConformanceTest{
 					},
 				},
 			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 18: request header transformer with split",
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host: "foo18.com",
+						Path: "/get",
+						RawHeaders: map[string][]string{
+							"X-split-dedupe-first": {"1,2,3"},
+							"X-split-dedupe-last":  {"a,b,c"},
+						},
+					},
+					ExpectedRequest: &http.ExpectedRequest{
+						Request: http.Request{
+							Host: "foo18.com",
+							Path: "/get",
+							Headers: map[string]string{
+								"X-split-dedupe-first": "1",
+								"X-split-dedupe-last":  "c",
+							},
+						},
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 200,
+					},
+				},
+			},
 		}
 		t.Run("WasmPlugin transformer", func(t *testing.T) {
 			for _, testcase := range testcases {

test/e2e/conformance/tests/go-wasm-transformer.yaml

@@ -400,6 +400,26 @@ spec:
                 port:
                   number: 8080
 ---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-header-split
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo18.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-v1
+                port:
+                  number: 8080
+---
 apiVersion: extensions.higress.io/v1alpha1
 kind: WasmPlugin
 metadata:
@@ -860,4 +880,15 @@ spec:
             headers:
               - key: reroute
                 newValue: true
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-header-split
+      configDisable: false
+      config:
+        reqRules:
+          - operate: dedupe
+            headers:
+              - key: X-split-dedupe-first
+                strategy: SPLIT_AND_RETAIN_FIRST
+              - key: X-split-dedupe-last
+                strategy: SPLIT_AND_RETAIN_LAST
   url: file:///opt/plugins/wasm-go/extensions/transformer/plugin.wasm

test/e2e/conformance/utils/http/http.go

@@ -93,6 +93,7 @@ type Request struct {
 	Method           string
 	Path             string
 	Headers          map[string]string
+	RawHeaders       http.Header
 	Body             []byte
 	ContentType      string
 	UnfollowRedirect bool
@@ -240,6 +241,14 @@ func MakeRequestAndExpectEventuallyConsistentResponse(t *testing.T, r roundtripp
 		}
 	}
 
+	if expected.Request.ActualRequest.RawHeaders != nil {
+		for name, values := range expected.Request.ActualRequest.RawHeaders {
+			for _, value := range values {
+				req.Headers[name] = append(req.Headers[name], strings.TrimSpace(value))
+			}
+		}
+	}
+
 	backendSetHeaders := make([]string, 0, len(expected.Response.AdditionalResponseHeaders))
 	for name, val := range expected.Response.AdditionalResponseHeaders {
 		backendSetHeaders = append(backendSetHeaders, name+":"+val)
@@ -755,7 +764,7 @@ func (er *Assertion) GetTestCaseName(i int) string {
 	headerStr := ""
 	reqStr := ""
 
-	if er.Request.ActualRequest.Headers != nil {
+	if er.Request.ActualRequest.Headers != nil || er.Request.ActualRequest.RawHeaders != nil {
 		headerStr = " with headers"
 	}