extension mechanism for custom logs and span attributes (#1451)

plugins/wasm-go/examples/custom-log/config.yaml

@@ -0,0 +1,68 @@
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        protocol: TCP
+        address: 0.0.0.0
+        port_value: 8080
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          access_log:
+          - name: envoy.access_loggers.file
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+              log_format:
+                text_format_source:
+                  inline_string: "{\"custom_log\":\"%FILTER_STATE(wasm.custom_log:PLAIN)%\",\"ai_log\":\"%FILTER_STATE(wasm.ai_log:PLAIN)%\"}
+                  
+                  "
+              path: /dev/stdout
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - name: get
+                match:
+                  prefix: "/get"
+                route:
+                  cluster: httpbin
+          http_filters:
+          - name: test
+            typed_config:
+              "@type": type.googleapis.com/udpa.type.v1.TypedStruct
+              type_url: type.googleapis.com/envoy.extensions.filters.http.wasm.v3.Wasm
+              value:
+                config:
+                  name: test
+                  vm_config:
+                    runtime: envoy.wasm.runtime.v8
+                    code:
+                      local:
+                        filename: main.wasm
+                  configuration:
+                    "@type": "type.googleapis.com/google.protobuf.StringValue"
+                    value: {}
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+  clusters:
+  - name: httpbin
+    connect_timeout: 600s
+    type: STRICT_DNS
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: httpbin
+      endpoints:
+        - lb_endpoints:
+          - endpoint:
+              address:
+                socket_address:
+                  address: httpbin.org
+                  port_value: 80

plugins/wasm-go/examples/custom-log/go.mod

@@ -0,0 +1,20 @@
+module github.com/alibaba/higress/plugins/wasm-go/extensions/custom-logs
+
+go 1.18
+
+replace github.com/alibaba/higress/plugins/wasm-go => ../..
+
+require (
+	github.com/alibaba/higress/plugins/wasm-go v0.0.0
+	github.com/higress-group/proxy-wasm-go-sdk v1.0.0
+)
+
+require (
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect
+	github.com/magefile/mage v1.14.0 // indirect
+	github.com/tidwall/gjson v1.17.3 // indirect
+	github.com/tidwall/match v1.1.1 // indirect
+	github.com/tidwall/pretty v1.2.0 // indirect
+	github.com/tidwall/resp v0.1.1 // indirect
+)

plugins/wasm-go/examples/custom-log/go.sum

@@ -0,0 +1,20 @@
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 h1:IHDghbGQ2DTIXHBHxWfqCYQW1fKjyJ/I7W1pMyUDeEA=
+github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520/go.mod h1:Nz8ORLaFiLWotg6GeKlJMhv8cci8mM43uEnLA5t8iew=
+github.com/higress-group/proxy-wasm-go-sdk v1.0.0 h1:BZRNf4R7jr9hwRivg/E29nkVaKEak5MWjBDhWjuHijU=
+github.com/higress-group/proxy-wasm-go-sdk v1.0.0/go.mod h1:iiSyFbo+rAtbtGt/bsefv8GU57h9CCLYGJA74/tF5/0=
+github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
+github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94=
+github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
+github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
+github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
+github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE=
+github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

plugins/wasm-go/examples/custom-log/main.go

@@ -0,0 +1,67 @@
+// Copyright (c) 2022 Alibaba Group Holding Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"math/rand"
+
+	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
+
+	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
+)
+
+func main() {
+	wrapper.SetCtx(
+		"custom-log",
+		wrapper.ProcessRequestHeadersBy(onHttpRequestHeaders),
+	)
+}
+
+type CustomLogConfig struct {
+}
+
+// Method 1: write custom log
+func writeLog(ctx wrapper.HttpContext) {
+	ctx.SetUserAttribute("question", "当然可以。在Python中，你可以创建一个函数来计算一系列数字的和。下面是一个简单的例子，该函数接受一个数字列表作为输入，并返回它们的总和。\n\n```python\ndef sum_of_numbers(numbers):\n    \"\"\"\n    计算列表中所有数字的和。\n    \n    参数:\n    numbers (list of int or float): 一个包含数字的列表。\n    \n    返回:\n    int or float: 列表中所有数字的总和。\n    \"\"\"\n    total_sum = sum(numbers)  # 使用Python内置的sum函数计算总和\n    return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers(numbers_list))  # 输出：The sum is: 15\n```\n\n在这段代码中，我们定义了一个名为 `sum_of_numbers` 的函数，它接收一个参数 `numbers`，这是一个包含整数或浮点数的列表。函数内部使用了Python的内置函数 `sum()` 来计算这些数字的总和，并将结果返回。\n\n你也可以手动实现求和逻辑，而不是使用内置的 `sum()` 函数，如下所示：\n\n```python\ndef sum_of_numbers_manual(numbers):\n    \"\"\"\n    手动计算列表中所有数字的和。\n    \n    参数:\n    numbers (list of int or float): 一个包含数字的列表。\n    \n    返回:\n    int or float: 列表中所有数字的总和。\n    \"\"\"\n    total_sum = 0\n    for number in numbers:\n        total_sum += number\n    return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers_manual(numbers_list))  # 输出：The sum is: 15\n```\n\n在这个版本中，我们初始化 `total_sum` 为0，然后遍历列表中的每个元素，并将其加到 `total_sum` 上。最后返回这个累加的结果。这两种方法都可以达到相同的目的，但是使用内置函数通常更简洁且效率更高。")
+	ctx.SetUserAttribute("k2", 2213.22)
+	ctx.WriteUserAttributeToLog()
+}
+
+// Methods 2: write custom log with specific key
+func writeLogWithKey(ctx wrapper.HttpContext, key string) {
+	ctx.SetUserAttribute("k2", 2213.22)
+	_ = ctx.WriteUserAttributeToLogWithKey(key)
+	ctx.SetUserAttribute("k2", 212939.22)
+	ctx.SetUserAttribute("k3", 123)
+	_ = ctx.WriteUserAttributeToLogWithKey(key)
+}
+
+// Methods 2: write custom log with specific key
+func writeTraceAttribute(ctx wrapper.HttpContext) {
+	ctx.SetUserAttribute("question", "当然可以。在Python中，你可以创建一个函数来计算一系列数字的和。下面是一个简单的例子，该函数接受一个数字列表作为输入，并返回它们的总和。\n\n```python\ndef sum_of_numbers(numbers):\n    \"\"\"\n    计算列表中所有数字的和。\n    \n    参数:\n    numbers (list of int or float): 一个包含数字的列表。\n    \n    返回:\n    int or float: 列表中所有数字的总和。\n    \"\"\"\n    total_sum = sum(numbers)  # 使用Python内置的sum函数计算总和\n    return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers(numbers_list))  # 输出：The sum is: 15\n```\n\n在这段代码中，我们定义了一个名为 `sum_of_numbers` 的函数，它接收一个参数 `numbers`，这是一个包含整数或浮点数的列表。函数内部使用了Python的内置函数 `sum()` 来计算这些数字的总和，并将结果返回。\n\n你也可以手动实现求和逻辑，而不是使用内置的 `sum()` 函数，如下所示：\n\n```python\ndef sum_of_numbers_manual(numbers):\n    \"\"\"\n    手动计算列表中所有数字的和。\n    \n    参数:\n    numbers (list of int or float): 一个包含数字的列表。\n    \n    返回:\n    int or float: 列表中所有数字的总和。\n    \"\"\"\n    total_sum = 0\n    for number in numbers:\n        total_sum += number\n    return total_sum\n\n# 示例使用\nnumbers_list = [1, 2, 3, 4, 5]\nprint(\"The sum is:\", sum_of_numbers_manual(numbers_list))  # 输出：The sum is: 15\n```\n\n在这个版本中，我们初始化 `total_sum` 为0，然后遍历列表中的每个元素，并将其加到 `total_sum` 上。最后返回这个累加的结果。这两种方法都可以达到相同的目的，但是使用内置函数通常更简洁且效率更高。")
+	ctx.SetUserAttribute("k2", 2213.22)
+	ctx.WriteUserAttributeToTrace()
+}
+
+func onHttpRequestHeaders(ctx wrapper.HttpContext, config CustomLogConfig, log wrapper.Log) types.Action {
+	if rand.Intn(10)%3 == 1 {
+		writeLog(ctx)
+	} else if rand.Intn(10)%3 == 2 {
+		writeLogWithKey(ctx, "ai_log")
+	} else {
+		writeTraceAttribute(ctx)
+	}
+	return types.ActionContinue
+}

plugins/wasm-go/pkg/wrapper/plugin_wrapper.go

@@ -15,6 +15,8 @@
 package wrapper
 
 import (
+	"encoding/json"
+	"fmt"
 	"strconv"
 	"time"
 	"unsafe"
@@ -26,6 +28,12 @@ import (
 	"github.com/alibaba/higress/plugins/wasm-go/pkg/matcher"
 )
 
+const (
+	CustomLogKey       = "custom_log"
+	AILogKey           = "ai_log"
+	TraceSpanTagPrefix = "trace_span_tag."
+)
+
 type HttpContext interface {
 	Scheme() string
 	Host() string
@@ -35,6 +43,14 @@ type HttpContext interface {
 	GetContext(key string) interface{}
 	GetBoolContext(key string, defaultValue bool) bool
 	GetStringContext(key, defaultValue string) string
+	GetUserAttribute(key string) interface{}
+	SetUserAttribute(key string, value interface{})
+	// You can call this function to set custom log
+	WriteUserAttributeToLog() error
+	// You can call this function to set custom log with your specific key
+	WriteUserAttributeToLogWithKey(key string) error
+	// You can call this function to set custom trace span attribute
+	WriteUserAttributeToTrace() error
 	// If the onHttpRequestBody handle is not set, the request body will not be read by default
 	DontReadRequestBody()
 	// If the onHttpResponseBody handle is not set, the request body will not be read by default
@@ -335,9 +351,10 @@ func (ctx *CommonPluginCtx[PluginConfig]) OnTick() {
 
 func (ctx *CommonPluginCtx[PluginConfig]) NewHttpContext(contextID uint32) types.HttpContext {
 	httpCtx := &CommonHttpCtx[PluginConfig]{
-		plugin:      ctx,
-		contextID:   contextID,
-		userContext: map[string]interface{}{},
+		plugin:        ctx,
+		contextID:     contextID,
+		userContext:   map[string]interface{}{},
+		userAttribute: map[string]interface{}{},
 	}
 	if ctx.vm.onHttpRequestBody != nil || ctx.vm.onHttpStreamingRequestBody != nil {
 		httpCtx.needRequestBody = true
@@ -367,6 +384,7 @@ type CommonHttpCtx[PluginConfig any] struct {
 	responseBodySize      int
 	contextID             uint32
 	userContext           map[string]interface{}
+	userAttribute         map[string]interface{}
 }
 
 func (ctx *CommonHttpCtx[PluginConfig]) SetContext(key string, value interface{}) {
@@ -377,6 +395,63 @@ func (ctx *CommonHttpCtx[PluginConfig]) GetContext(key string) interface{} {
 	return ctx.userContext[key]
 }
 
+func (ctx *CommonHttpCtx[PluginConfig]) SetUserAttribute(key string, value interface{}) {
+	ctx.userAttribute[key] = value
+}
+
+func (ctx *CommonHttpCtx[PluginConfig]) GetUserAttribute(key string) interface{} {
+	return ctx.userAttribute[key]
+}
+
+func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToLog() error {
+	return ctx.WriteUserAttributeToLogWithKey(CustomLogKey)
+}
+
+func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToLogWithKey(key string) error {
+	// e.g. {\"field1\":\"value1\",\"field2\":\"value2\"}
+	preMarshalledJsonLogStr, _ := proxywasm.GetProperty([]string{key})
+	newAttributeMap := map[string]interface{}{}
+	if string(preMarshalledJsonLogStr) != "" {
+		// e.g. {"field1":"value1","field2":"value2"}
+		preJsonLogStr := unmarshalStr(fmt.Sprintf(`"%s"`, string(preMarshalledJsonLogStr)))
+		err := json.Unmarshal([]byte(preJsonLogStr), &newAttributeMap)
+		if err != nil {
+			ctx.plugin.vm.log.Warnf("Unmarshal failed, will overwrite %s, pre value is: %s", key, string(preMarshalledJsonLogStr))
+			return err
+		}
+	}
+	// update customLog
+	for k, v := range ctx.userAttribute {
+		newAttributeMap[k] = v
+	}
+	// e.g. {"field1":"value1","field2":2,"field3":"value3"}
+	jsonStr, _ := json.Marshal(newAttributeMap)
+	// e.g. {\"field1\":\"value1\",\"field2\":2,\"field3\":\"value3\"}
+	marshalledJsonStr := marshalStr(string(jsonStr))
+	if err := proxywasm.SetProperty([]string{key}, []byte(marshalledJsonStr)); err != nil {
+		ctx.plugin.vm.log.Warnf("failed to set %s in filter state, raw is %s, err is %v", key, marshalledJsonStr, err)
+		return err
+	}
+	return nil
+}
+
+func (ctx *CommonHttpCtx[PluginConfig]) WriteUserAttributeToTrace() error {
+	for k, v := range ctx.userAttribute {
+		traceSpanTag := TraceSpanTagPrefix + k
+		traceSpanValue := fmt.Sprint(v)
+		var err error
+		if traceSpanValue != "" {
+			err = proxywasm.SetProperty([]string{traceSpanTag}, []byte(traceSpanValue))
+		} else {
+			err = fmt.Errorf("value of %s is empty", traceSpanTag)
+		}
+		if err != nil {
+			ctx.plugin.vm.log.Warnf("Failed to set trace attribute - %s: %s, error message: %v", traceSpanTag, traceSpanValue, err)
+		}
+	}
+	return nil
+}
+
 func (ctx *CommonHttpCtx[PluginConfig]) GetBoolContext(key string, defaultValue bool) bool {
 	if b, ok := ctx.userContext[key].(bool); ok {
 		return b

plugins/wasm-go/pkg/wrapper/utils.go

@@ -0,0 +1,36 @@
+package wrapper
+
+import (
+	"encoding/json"
+
+	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
+	"github.com/tidwall/gjson"
+)
+
+func unmarshalStr(marshalledJsonStr string) string {
+	// e.g. "{\"field1\":\"value1\",\"field2\":\"value2\"}"
+	var jsonStr string
+	err := json.Unmarshal([]byte(marshalledJsonStr), &jsonStr)
+	if err != nil {
+		proxywasm.LogErrorf("failed to unmarshal json string, raw string is: %s, err is: %v", marshalledJsonStr, err)
+		return ""
+	}
+	// e.g. {"field1":"value1","field2":"value2"}
+	return jsonStr
+}
+
+func marshalStr(raw string) string {
+	// e.g. {"field1":"value1","field2":"value2"}
+	helper := map[string]string{
+		"placeholder": raw,
+	}
+	marshalledHelper, _ := json.Marshal(helper)
+	marshalledRaw := gjson.GetBytes(marshalledHelper, "placeholder").Raw
+	if len(marshalledRaw) >= 2 {
+		// e.g. {\"field1\":\"value1\",\"field2\":\"value2\"}
+		return marshalledRaw[1 : len(marshalledRaw)-1]
+	} else {
+		proxywasm.LogErrorf("failed to marshal json string, raw string is: %s", raw)
+		return ""
+	}
+}