diff --git a/plugins/wasm-go/extensions/ai-proxy/provider/provider.go b/plugins/wasm-go/extensions/ai-proxy/provider/provider.go index 563f868a6..69575df6b 100644 --- a/plugins/wasm-go/extensions/ai-proxy/provider/provider.go +++ b/plugins/wasm-go/extensions/ai-proxy/provider/provider.go @@ -61,6 +61,8 @@ const ( // TODO: 以下是一些非标准的API名称，需要进一步确认是否支持 ApiNameCohereV1Rerank ApiName = "cohere/v1/rerank" + ApiNameQwenAsyncAIGC ApiName = "api/v1/services/aigc" + ApiNameQwenAsyncTask ApiName = "api/v1/tasks/" providerTypeMoonshot = "moonshot" providerTypeAzure = "azure" diff --git a/plugins/wasm-go/extensions/ai-proxy/provider/qwen.go b/plugins/wasm-go/extensions/ai-proxy/provider/qwen.go index 7a3e7d200..0fa2de085 100644 --- a/plugins/wasm-go/extensions/ai-proxy/provider/qwen.go +++ b/plugins/wasm-go/extensions/ai-proxy/provider/qwen.go @@ -37,6 +37,9 @@ const ( qwenBailianPath = "/api/v1/apps" qwenMultimodalGenerationPath = "/api/v1/services/aigc/multimodal-generation/generation" + qwenAsyncAIGCPath = "/api/v1/services/aigc/" + qwenAsyncTaskPath = "/api/v1/tasks/" + qwenTopPMin = 0.000001 qwenTopPMax = 0.999999 @@ -74,6 +77,8 @@ func (m *qwenProviderInitializer) DefaultCapabilities(qwenEnableCompatible bool) return map[string]string{ string(ApiNameChatCompletion): qwenChatCompletionPath, string(ApiNameEmbeddings): qwenTextEmbeddingPath, + string(ApiNameQwenAsyncAIGC): qwenAsyncAIGCPath, + string(ApiNameQwenAsyncTask): qwenAsyncTaskPath, } } } @@ -689,6 +694,10 @@ func (m *qwenProvider) GetApiName(path string) ApiName { case strings.Contains(path, qwenTextEmbeddingPath), strings.Contains(path, qwenCompatibleTextEmbeddingPath): return ApiNameEmbeddings + case strings.Contains(path, qwenAsyncAIGCPath): + return ApiNameQwenAsyncAIGC + case strings.Contains(path, qwenAsyncTaskPath): + return ApiNameQwenAsyncTask default: return "" } diff --git a/plugins/wasm-go/extensions/ai-statistics/README.md b/plugins/wasm-go/extensions/ai-statistics/README.md index 7d8c07969..294a51469 100644 --- a/plugins/wasm-go/extensions/ai-statistics/README.md +++ b/plugins/wasm-go/extensions/ai-statistics/README.md @@ -23,6 +23,7 @@ description: AI可观测配置参考 | 名称 | 数据类型 | 填写要求 | 默认值 | 描述 | |----------------|-------|------|-----|------------------------| | `attributes` | []Attribute | 非必填 | - | 用户希望记录在log/span中的信息 | +| `disable_openai_usage` | bool | 非必填 | false | 非openai兼容协议时，model、token的支持非标，配置为true时可以避免报错 | Attribute 配置说明: diff --git a/plugins/wasm-go/extensions/ai-statistics/README_EN.md b/plugins/wasm-go/extensions/ai-statistics/README_EN.md index 63c5cfc2b..86a76c2cf 100644 --- a/plugins/wasm-go/extensions/ai-statistics/README_EN.md +++ b/plugins/wasm-go/extensions/ai-statistics/README_EN.md @@ -22,7 +22,8 @@ Users can also expand observable values ​​through configuration: | Name | Type | Required | Default | Description | |----------------|-------|------|-----|------------------------| -| `attributes` | []Attribute | required | - | Information that the user wants to record in log/span | +| `attributes` | []Attribute | optional | - | Information that the user wants to record in log/span | +| `disable_openai_usage` | bool | optional | false | When using a non-OpenAI-compatible protocol, the support for model and token is non-standard. Setting the configuration to true can prevent errors. | Attribute Configuration instructions: diff --git a/plugins/wasm-go/extensions/ai-statistics/main.go b/plugins/wasm-go/extensions/ai-statistics/main.go index 5ad094dbc..d9d10c4fe 100644 --- a/plugins/wasm-go/extensions/ai-statistics/main.go +++ b/plugins/wasm-go/extensions/ai-statistics/main.go @@ -92,6 +92,8 @@ type AIStatisticsConfig struct { attributes []Attribute // If there exist attributes extracted from streaming body, chunks should be buffered shouldBufferStreamingBody bool + // If disableOpenaiUsage is true, model/input_token/output_token logs will be skipped + disableOpenaiUsage bool } func generateMetricName(route, cluster, model, consumer, metricName string) string { @@ -160,6 +162,10 @@ func parseConfig(configJson gjson.Result, config *AIStatisticsConfig, log wrappe } // Metric settings config.counterMetrics = make(map[string]proxywasm.MetricCounter) + + // Parse openai usage config setting. + config.disableOpenaiUsage = configJson.Get("disable_openai_usage").Bool() + return nil } @@ -264,15 +270,17 @@ func onHttpStreamingBody(ctx wrapper.HttpContext, config AIStatisticsConfig, dat } // Set information about this request - if model, inputToken, outputToken, ok := getUsage(data); ok { - ctx.SetUserAttribute(Model, model) - ctx.SetUserAttribute(InputToken, inputToken) - ctx.SetUserAttribute(OutputToken, outputToken) - // Set span attributes for ARMS. - setSpanAttribute(ArmsModelName, model, log) - setSpanAttribute(ArmsInputToken, inputToken, log) - setSpanAttribute(ArmsOutputToken, outputToken, log) - setSpanAttribute(ArmsTotalToken, inputToken+outputToken, log) + if !config.disableOpenaiUsage { + if model, inputToken, outputToken, ok := getUsage(data); ok { + ctx.SetUserAttribute(Model, model) + ctx.SetUserAttribute(InputToken, inputToken) + ctx.SetUserAttribute(OutputToken, outputToken) + // Set span attributes for ARMS. + setSpanAttribute(ArmsModelName, model, log) + setSpanAttribute(ArmsInputToken, inputToken, log) + setSpanAttribute(ArmsOutputToken, outputToken, log) + setSpanAttribute(ArmsTotalToken, inputToken+outputToken, log) + } } // If the end of the stream is reached, record metrics/logs/spans. if endOfStream { @@ -311,15 +319,17 @@ func onHttpResponseBody(ctx wrapper.HttpContext, config AIStatisticsConfig, body } // Set information about this request - if model, inputToken, outputToken, ok := getUsage(body); ok { - ctx.SetUserAttribute(Model, model) - ctx.SetUserAttribute(InputToken, inputToken) - ctx.SetUserAttribute(OutputToken, outputToken) - // Set span attributes for ARMS. - setSpanAttribute(ArmsModelName, model, log) - setSpanAttribute(ArmsInputToken, inputToken, log) - setSpanAttribute(ArmsOutputToken, outputToken, log) - setSpanAttribute(ArmsTotalToken, inputToken+outputToken, log) + if !config.disableOpenaiUsage { + if model, inputToken, outputToken, ok := getUsage(body); ok { + ctx.SetUserAttribute(Model, model) + ctx.SetUserAttribute(InputToken, inputToken) + ctx.SetUserAttribute(OutputToken, outputToken) + // Set span attributes for ARMS. + setSpanAttribute(ArmsModelName, model, log) + setSpanAttribute(ArmsInputToken, inputToken, log) + setSpanAttribute(ArmsOutputToken, outputToken, log) + setSpanAttribute(ArmsTotalToken, inputToken+outputToken, log) + } } // Set user defined log & span attributes. @@ -471,6 +481,11 @@ func writeMetric(ctx wrapper.HttpContext, config AIStatisticsConfig, log wrapper log.Warnf("ClusterName typd assert failed, skip metric record") return } + + if config.disableOpenaiUsage { + return + } + if ctx.GetUserAttribute(Model) == nil || ctx.GetUserAttribute(InputToken) == nil || ctx.GetUserAttribute(OutputToken) == nil { log.Warnf("get usage information failed, skip metric record") return