fix(wasm-go): transformer performs an add op when the replace key does not exist (#2706)

plugins/wasm-go/extensions/transformer/VERSION

@@ -1 +1 @@
-1.0.0
+1.0.1-alpha

plugins/wasm-go/extensions/transformer/docker-compose.yaml

@@ -0,0 +1,26 @@
+version: '3.7'
+services:
+  envoy:
+    image: higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/gateway:v2.0.7
+    entrypoint: /usr/local/bin/envoy
+    # 注意这里对wasm开启了debug级别日志，正式部署时则默认info级别
+    command: -c /etc/envoy/envoy.yaml --component-log-level wasm:debug
+    #depends_on:
+    #  - httpbin
+    networks:
+      - wasmtest
+    ports:
+      - "10000:10000"
+    volumes:
+      - ./envoy.yaml:/etc/envoy/envoy.yaml
+      - ./plugin.wasm:/etc/envoy/main.wasm
+
+  httpbin:
+    image: kong/httpbin:latest
+    networks:
+      - wasmtest
+    ports:
+      - "12345:80"
+
+networks:
+  wasmtest: {}

plugins/wasm-go/extensions/transformer/envoy.yaml

@@ -0,0 +1,92 @@
+admin:
+  address:
+    socket_address:
+      protocol: TCP
+      address: 0.0.0.0
+      port_value: 9901
+static_resources:
+  listeners:
+    - name: listener_0
+      address:
+        socket_address:
+          protocol: TCP
+          address: 0.0.0.0
+          port_value: 10000
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                scheme_header_transformation:
+                  scheme_to_overwrite: https
+                stat_prefix: ingress_http
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: local_service
+                      domains: ["*"]
+                      routes:
+                        - match:
+                            prefix: "/"
+                          route:
+                            cluster: httpbin
+                http_filters:
+                  - name: wasmdemo
+                    typed_config:
+                      "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+                      type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
+                      value:
+                        config:
+                          name: wasmdemo
+                          vm_config:
+                            runtime: envoy.wasm.runtime.v8
+                            code:
+                              local:
+                                filename: /etc/envoy/main.wasm
+                          configuration:
+                            "@type": "type.googleapis.com/google.protobuf.StringValue"
+                            value: |
+                              {
+                                "reqRules": [
+                                  {
+                                    "operate": "replace",
+                                    "headers": [
+                                      {
+                                        "key": "hello",
+                                        "newValue": "higress"
+                                      }
+                                    ],
+                                    "querys": [
+                                      {
+                                        "key": "k1",
+                                        "newValue": "newQueryV1"
+                                      }
+                                    ],
+                                    "body": [
+                                      {
+                                        "key": "k2",
+                                        "newValue": "newBodyV2"
+                                      }
+                                    ]
+                                  }
+                                ]
+                              }
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+  clusters:
+    - name: httpbin
+      connect_timeout: 30s
+      type: LOGICAL_DNS
+      # Comment out the following line to test on v6 networks
+      dns_lookup_family: V4_ONLY
+      lb_policy: ROUND_ROBIN
+      load_assignment:
+        cluster_name: httpbin
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: httpbin
+                      port_value: 80

plugins/wasm-go/extensions/transformer/main.go

@@ -915,16 +915,13 @@ func (h kvHandler) handle(host, path string, kvs map[string][]string, mapSourceD
 				}
 			}
 		case ReplaceK:
-			// replace: 若指定 key 不存在，则无操作；否则替换 value 为 newValue
+			// replace: 若指定 key 不存在，相当于添加操作；否则替换 value 为 newValue
 			for _, replace := range kvtOp.replaceKvtGroup {
 				key, newValue := replace.key, replace.newValue
-				if _, ok := kvs[key]; !ok {
-					continue
-				}
 				if replace.reg != nil {
 					newValue = replace.reg.matchAndReplace(newValue, host, path)
 				}
-				kvs[replace.key] = []string{newValue}
+				kvs[key] = []string{newValue}
 			}
 		case AddK:
 			// add: 若指定 key 存在则无操作；否则添加 key:value
@@ -1047,12 +1044,9 @@ func (h jsonHandler) handle(host, path string, oriData []byte, mapSourceData map
 				}
 			}
 		case ReplaceK:
-			// replace: 若指定 key 不存在，则无操作；否则替换 value 为 newValue
+			// replace: 若指定 key 不存在，则相当于添加操作；否则替换 value 为 newValue
 			for _, replace := range kvtOp.replaceKvtGroup {
 				key, newValue, valueType := replace.key, replace.newValue, replace.typ
-				if !gjson.GetBytes(data, key).Exists() {
-					continue
-				}
 				if valueType == "string" && replace.reg != nil {
 					newValue = replace.reg.matchAndReplace(newValue, host, path)
 				}

test/e2e/conformance/tests/go-wasm-transformer.go

@@ -259,7 +259,6 @@ var WasmPluginsTransformer = suite.ConformanceTest{
 								"X-map": "vmap",
 							},
 						},
-
 					},
 				},
 				Response: http.AssertionResponse{
@@ -315,7 +314,8 @@ var WasmPluginsTransformer = suite.ConformanceTest{
 						{
 							"X-removed":["v1", "v2"],
 							"X-not-renamed":["v1"],
-							"X-to-be-mapped":["v1", "v2"]
+							"X-to-be-mapped":["v1", "v2"],
+							"X-replace": "not-replaced"
 						}
 						`),
 						ContentType: http.ContentTypeApplicationJson,
@@ -332,7 +332,8 @@ var WasmPluginsTransformer = suite.ConformanceTest{
 								"X-renamed":["v1"],
 								"X-add-append":["add","append"],
 								"X-to-be-mapped":["v1", "v2"],
-								"X-map":["v1", "v2"]
+								"X-map":["v1", "v2"],
+								"X-replace": "replaced"
 							}
 						`),
 						},
@@ -376,7 +377,199 @@ var WasmPluginsTransformer = suite.ConformanceTest{
 							"X-renamed":["v1"],
 							"X-add-append":["add","append"],
 							"X-to-be-mapped":["v1", "v2"],
-							"X-map":["v1", "v2"]
+							"X-map":["v1", "v2"],
+							"X-replace":"replaced"
+						}
+						`),
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 10: map from headers to body",
+					TargetBackend:   "infra-backend-echo-body-v1",
+					TargetNamespace: "higress-conformance-infra",
+					CompareTarget:   http.CompareTargetResponse,
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:    "foo10.com",
+						Path:    "/post",
+						Method:  "POST",
+						Headers: map[string]string{"X-map": "higress"},
+						Body: []byte(`
+						{
+							"X-hello":"world"
+						}
+						`),
+						ContentType: http.ContentTypeApplicationJson,
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode:  200,
+						ContentType: http.ContentTypeApplicationJson,
+						Body: []byte(`
+						{
+							"X-hello":"world",
+							"kmap":["higress"]
+						}
+						`),
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 11: map from querys to body",
+					TargetBackend:   "infra-backend-echo-body-v1",
+					TargetNamespace: "higress-conformance-infra",
+					CompareTarget:   http.CompareTargetResponse,
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:   "foo11.com",
+						Path:   "/post?X-map=higress",
+						Method: "POST",
+						Body: []byte(`
+						{
+							"X-hello": "world"
+						}
+						`),
+						ContentType: http.ContentTypeApplicationJson,
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode:  200,
+						ContentType: http.ContentTypeApplicationJson,
+						Body: []byte(`
+						{
+							"X-hello": "world",
+							"test": {
+								"kmap": ["higress"]
+							}
+						}
+						`),
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 12: map from body to headers",
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:   "foo12.com",
+						Path:   "/post",
+						Method: "POST",
+						Body: []byte(`
+						{
+							"test": {
+								"kmap": "higress"
+							}
+						}
+						`),
+						ContentType: http.ContentTypeApplicationJson,
+					},
+					ExpectedRequest: &http.ExpectedRequest{
+						Request: http.Request{
+							Host:    "foo12.com",
+							Path:    "/post",
+							Method:  "POST",
+							Headers: map[string]string{"X-map": "higress"},
+						},
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 200,
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 13: map from body to querys",
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:   "foo13.com",
+						Path:   "/post",
+						Method: "POST",
+						Body: []byte(`
+						{
+							"test": {
+								"kmap": "higress"
+							}
+						}
+						`),
+						ContentType: http.ContentTypeApplicationJson,
+					},
+					ExpectedRequest: &http.ExpectedRequest{
+						Request: http.Request{
+							Host:   "foo13.com",
+							Path:   "/post?X-map=higress",
+							Method: "POST",
+						},
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 200,
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 14: headers & querys, when replace key is not exist, it is equivalent to app",
+					TargetBackend:   "infra-backend-v1",
+					TargetNamespace: "higress-conformance-infra",
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host: "foo14.com",
+						Path: "/get?X-replace-querys=hello",
+					},
+					ExpectedRequest: &http.ExpectedRequest{
+						Request: http.Request{
+							Host:    "foo14.com",
+							Path:    "/get?X-replace-querys=exist-querys",
+							Headers: map[string]string{"X-replace-headers": "exist-headers"},
+						},
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode: 200,
+					},
+				},
+			},
+			{
+				Meta: http.AssertionMeta{
+					TestCaseName:    "case 15: body, when replace key is not exist, it is equivalent to add",
+					TargetBackend:   "infra-backend-echo-body-v1",
+					TargetNamespace: "higress-conformance-infra",
+					CompareTarget:   http.CompareTargetResponse,
+				},
+				Request: http.AssertionRequest{
+					ActualRequest: http.Request{
+						Host:        "foo15.com",
+						Path:        "/post",
+						Method:      "POST",
+						Body:        []byte(`{}`),
+						ContentType: http.ContentTypeApplicationJson,
+					},
+				},
+				Response: http.AssertionResponse{
+					ExpectedResponse: http.Response{
+						StatusCode:  200,
+						ContentType: http.ContentTypeApplicationJson,
+						Body: []byte(`
+						{
+							"X-replace-body": "exist-body"
 						}
 						`),
 					},

test/e2e/conformance/tests/go-wasm-transformer.yaml

@@ -112,8 +112,6 @@ spec:
                 port:
                   number: 8080
 ---
-
----
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
@@ -194,6 +192,126 @@ spec:
                 port:
                   number: 8080
 ---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-map-from-headers-to-body
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo10.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-echo-body-v1
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-map-from-querys-to-body
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo11.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-echo-body-v1
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-map-from-body-to-headers
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo12.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-v1
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-map-from-body-to-querys
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo13.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-v1
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-headers-querys-replace-is-not-exist
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo14.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-v1
+                port:
+                  number: 8080
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+  name: wasmplugin-transform-request-body-replace-is-not-exist
+  namespace: higress-conformance-infra
+spec:
+  ingressClassName: higress
+  rules:
+    - host: "foo15.com"
+      http:
+        paths:
+          - pathType: Prefix
+            path: "/"
+            backend:
+              service:
+                name: infra-backend-echo-body-v1
+                port:
+                  number: 8080
+---
 apiVersion: extensions.higress.io/v1alpha1
 kind: WasmPlugin
 metadata:
@@ -534,36 +652,103 @@ spec:
         - higress-conformance-infra/wasmplugin-transform-response-body
       configDisable: false
       config:
-        respRules:
-          - operate: remove
+          respRules:
+            - operate: remove
+              body:
+                - key: X-removed
+            - operate: rename
+              body:
+                - oldKey: X-not-renamed
+                  newKey: X-renamed
+            - operate: replace
+              body:
+                - key: X-replace
+                  newValue: replaced
+            - operate: add
+              body:
+                - key: X-add-append
+                  value: add
+            - operate: append
+              body:
+                - key: X-add-append
+                  appendValue: append
+            - operate: map
+              body:
+                - fromKey: X-to-be-mapped
+                  toKey: X-map
+            - operate: dedupe
+              body:
+                - key: X-dedupe-first
+                  strategy: RETAIN_FIRST
+                - key: X-dedupe-last
+                  strategy: RETAIN_LAST
+                - key: X-dedupe-unique
+                  strategy: RETAIN_UNIQUE
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-map-from-headers-to-body
+      configDisable: false
+      config:
+        reqRules:
+          - operate: map
+            mapSource: headers
             body:
-              - key: X-removed
-          - operate: rename
+              - fromKey: X-map
+                toKey: kmap
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-map-from-querys-to-body
+      configDisable: false
+      config:
+        reqRules:
+          - operate: map
+            mapSource: querys
             body:
-              - oldKey: X-not-renamed
-                newKey: X-renamed
+              - fromKey: X-map
+                toKey: test.kmap
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-map-from-body-to-headers
+      configDisable: false
+      config:
+        reqRules:
+          - operate: map
+            mapSource: body
+            headers:
+              - fromKey: test.kmap
+                toKey: X-map
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-map-from-body-to-querys
+      configDisable: false
+      config:
+        reqRules:
+          - operate: map
+            mapSource: body
+            querys:
+              - fromKey: test.kmap
+                toKey: X-map
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-headers-querys-replace-is-not-exist
+      configDisable: false
+      config:
+        reqRules:
+          - operate: replace
+            headers:
+              - key: X-replace-headers
+                newValue: exist-headers
+            querys:
+              - key: X-replace-querys
+                newValue: exist-querys
+
+    - ingress:
+        - higress-conformance-infra/wasmplugin-transform-request-body-replace-is-not-exist
+      configDisable: false
+      config:
+        reqRules:
           - operate: replace
             body:
-              - key: X-replace
-                newValue: replaced
-          - operate: add
-            body:
-              - key: X-add-append
-                value: add
-          - operate: append
-            body:
-              - key: X-add-append
-                appendValue: append
-          - operate: map
-            body:
-              - fromKey: X-to-be-mapped
-                toKey: X-map
-          - operate: dedupe
-            body:
-              - key: X-dedupe-first
-                strategy: RETAIN_FIRST
-              - key: X-dedupe-last
-                strategy: RETAIN_LAST
-              - key: X-dedupe-unique
-                strategy: RETAIN_UNIQUE
-  url: oci://higress-registry.cn-hangzhou.cr.aliyuncs.com/plugins/transformer:2.0.0
+              - key: X-replace-body
+                newValue: exist-body
+  url: file:///opt/plugins/wasm-go/extensions/transformer/plugin.wasm