feat: Adapt to the Qwen multimodal model generation API (#1221)

plugins/wasm-go/extensions/ai-proxy/README.md

@@ -318,6 +318,7 @@ provider:
     'gpt-35-turbo': "qwen-plus"
     'gpt-4-turbo': "qwen-max"
     'gpt-4-*': "qwen-max"
+    'gpt-4o': "qwen-vl-plus"
     'text-embedding-v1': 'text-embedding-v1'
     '*': "qwen-turbo"
 ```
@@ -326,7 +327,111 @@ provider:
 
 URL: http://your-domain/v1/chat/completions
 
-请求体：
+请求示例：
+
+```json
+{
+  "model": "gpt-3",
+  "messages": [
+    {
+      "role": "user",
+      "content": "你好，你是谁？"
+    }
+  ],
+  "temperature": 0.3
+}
+```
+
+响应示例：
+
+```json
+{
+  "id": "c2518bd3-0f46-97d1-be34-bb5777cb3108",
+  "choices": [
+    {
+      "index": 0,
+      "message": {
+        "role": "assistant",
+        "content": "我是通义千问，由阿里云开发的AI助手。我可以回答各种问题、提供信息和与用户进行对话。有什么我可以帮助你的吗？"
+      },
+      "finish_reason": "stop"
+    }
+  ],
+  "created": 1715175072,
+  "model": "qwen-turbo",
+  "object": "chat.completion",
+  "usage": {
+    "prompt_tokens": 24,
+    "completion_tokens": 33,
+    "total_tokens": 57
+  }
+}
+```
+
+**多模态模型 API 请求示例（适用于 `qwen-vl-plus` 和 `qwen-vl-max` 模型）**
+
+URL: http://your-domain/v1/chat/completions
+
+请求示例：
+
+```json
+{
+    "model": "gpt-4o",
+    "messages": [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": "https://dashscope.oss-cn-beijing.aliyuncs.com/images/dog_and_girl.jpeg"
+                    }
+                },
+                {
+                    "type": "text",
+                    "text": "这个图片是哪里？"
+                }
+            ]
+        }
+    ],
+    "temperature": 0.3
+}
+```
+
+响应示例：
+
+```json
+{
+    "id": "17c5955d-af9c-9f28-bbde-293a9c9a3515",
+    "choices": [
+        {
+            "index": 0,
+            "message": {
+                "role": "assistant",
+                "content": [
+                    {
+                        "text": "这张照片显示的是一位女士和一只狗在海滩上。由于我无法获取具体的地理位置信息，所以不能确定这是哪个地方的海滩。但是从视觉内容来看，它可能是一个位于沿海地区的沙滩海岸线，并且有海浪拍打着岸边。这样的场景在全球许多美丽的海滨地区都可以找到。如果您需要更精确的信息，请提供更多的背景或细节描述。"
+                    }
+                ]
+            },
+            "finish_reason": "stop"
+        }
+    ],
+    "created": 1723949230,
+    "model": "qwen-vl-plus",
+    "object": "chat.completion",
+    "usage": {
+        "prompt_tokens": 1279,
+        "completion_tokens": 78
+    }
+}
+```
+
+**文本向量请求示例**
+
+URL: http://your-domain/v1/embeddings
+
+请求示例：
 
 ```json
 {
@@ -335,7 +440,7 @@ URL: http://your-domain/v1/chat/completions
 }
 ```
 
-响应体示例：
+响应示例：
 
 ```json
 {
@@ -367,47 +472,6 @@ URL: http://your-domain/v1/chat/completions
 }
 ```
 
-**请求示例**
-
-URL: http://your-domain/v1/embeddings
-
-示例请求内容：
-
-```json
-{
-    "model": "text-embedding-v1",
-    "input": [
-        "Hello world!"
-    ]
-}
-```
-
-示例响应内容：
-
-```json
-{
-  "id": "c2518bd3-0f46-97d1-be34-bb5777cb3108",
-  "choices": [
-    {
-      "index": 0,
-      "message": {
-        "role": "assistant",
-        "content": "我是通义千问，由阿里云开发的AI助手。我可以回答各种问题、提供信息和与用户进行对话。有什么我可以帮助你的吗？"
-      },
-      "finish_reason": "stop"
-    }
-  ],
-  "created": 1715175072,
-  "model": "qwen-turbo",
-  "object": "chat.completion",
-  "usage": {
-    "prompt_tokens": 24,
-    "completion_tokens": 33,
-    "total_tokens": 57
-  }
-}
-```
-
 ### 使用通义千问配合纯文本上下文信息
 
 使用通义千问服务，同时配置纯文本上下文信息。

plugins/wasm-go/extensions/ai-proxy/go.mod

@@ -15,7 +15,7 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/uuid v1.3.0
 	github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect
 	github.com/magefile/mage v1.14.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

plugins/wasm-go/extensions/ai-proxy/provider/baidu.go

@@ -253,7 +253,7 @@ func (b *baiduProvider) baiduTextGenRequest(request *chatCompletionRequest) *bai
 	}
 	for _, message := range request.Messages {
 		if message.Role == roleSystem {
-			baiduRequest.System = message.Content
+			baiduRequest.System = message.StringContent()
 		} else {
 			baiduRequest.Messages = append(baiduRequest.Messages, chatMessage{
 				Role:    message.Role,

plugins/wasm-go/extensions/ai-proxy/provider/claude.go

@@ -274,7 +274,7 @@ func (c *claudeProvider) buildClaudeTextGenRequest(origRequest *chatCompletionRe
 
 	for _, message := range origRequest.Messages {
 		if message.Role == roleSystem {
-			claudeRequest.System = message.Content
+			claudeRequest.System = message.StringContent()
 			continue
 		}
 		claudeMessage := chatMessage{

plugins/wasm-go/extensions/ai-proxy/provider/deepl.go

@@ -114,9 +114,9 @@ func (d *deeplProvider) OnRequestBody(ctx wrapper.HttpContext, apiName ApiName,
 		}
 		for _, msg := range originRequest.Messages {
 			if msg.Role == roleSystem {
-				deeplRequest.Context = msg.Content
+				deeplRequest.Context = msg.StringContent()
 			} else {
-				deeplRequest.Text = append(deeplRequest.Text, msg.Content)
+				deeplRequest.Text = append(deeplRequest.Text, msg.StringContent())
 			}
 		}
 		return types.ActionContinue, replaceJsonRequestBody(deeplRequest, log)

plugins/wasm-go/extensions/ai-proxy/provider/gemini.go

@@ -4,13 +4,14 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strings"
+	"time"
+
 	"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
 	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
 	"github.com/google/uuid"
 	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
 	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
-	"strings"
-	"time"
 )
 
 // geminiProvider is the provider for google gemini/gemini flash service.
@@ -378,7 +379,7 @@ func (g *geminiProvider) buildGeminiChatRequest(request *chatCompletionRequest)
 			Role: message.Role,
 			Parts: []geminiPart{
 				{
-					Text: message.Content,
+					Text: message.StringContent(),
 				},
 			},
 		}

plugins/wasm-go/extensions/ai-proxy/provider/hunyuan.go

@@ -447,7 +447,7 @@ func convertMessagesFromOpenAIToHunyuan(openAIMessages []chatMessage) []hunyuanC
 	for _, msg := range openAIMessages {
 		hunyuanChatMessages = append(hunyuanChatMessages, hunyuanChatMessage{
 			Role:    msg.Role,
-			Content: msg.Content,
+			Content: msg.StringContent(),
 		})
 	}
 

plugins/wasm-go/extensions/ai-proxy/provider/minimax.go

@@ -404,19 +404,19 @@ func (m *minimaxProvider) buildMinimaxChatCompletionV2Request(request *chatCompl
 			botName = determineName(message.Name, defaultBotName)
 			botSetting = append(botSetting, minimaxBotSetting{
 				BotName: botName,
-				Content: message.Content,
+				Content: message.StringContent(),
 			})
 		case roleAssistant:
 			messages = append(messages, minimaxMessage{
 				SenderType: senderTypeBot,
 				SenderName: determineName(message.Name, defaultBotName),
-				Text:       message.Content,
+				Text:       message.StringContent(),
 			})
 		case roleUser:
 			messages = append(messages, minimaxMessage{
 				SenderType: senderTypeUser,
 				SenderName: determineName(message.Name, defaultSenderName),
-				Text:       message.Content,
+				Text:       message.StringContent(),
 			})
 		}
 	}

plugins/wasm-go/extensions/ai-proxy/provider/model.go

@@ -13,6 +13,9 @@ const (
 	eventResult = "result"
 
 	httpStatus200 = "200"
+
+	contentTypeText     = "text"
+	contentTypeImageUrl = "image_url"
 )
 
 type chatCompletionRequest struct {
@@ -80,12 +83,27 @@ type usage struct {
 type chatMessage struct {
 	Name      string     `json:"name,omitempty"`
 	Role      string     `json:"role,omitempty"`
-	Content   string     `json:"content,omitempty"`
+	Content   any        `json:"content,omitempty"`
 	ToolCalls []toolCall `json:"tool_calls,omitempty"`
 }
 
+type messageContent struct {
+	Type     string    `json:"type,omitempty"`
+	Text     string    `json:"text"`
+	ImageUrl *imageUrl `json:"image_url,omitempty"`
+}
+
+type imageUrl struct {
+	Url    string `json:"url,omitempty"`
+	Detail string `json:"detail,omitempty"`
+}
+
 func (m *chatMessage) IsEmpty() bool {
-	if m.Content != "" {
+	if m.IsStringContent() && m.Content != "" {
+		return false
+	}
+	anyList, ok := m.Content.([]any)
+	if ok && len(anyList) > 0 {
 		return false
 	}
 	if len(m.ToolCalls) != 0 {
@@ -103,6 +121,76 @@ func (m *chatMessage) IsEmpty() bool {
 	return true
 }
 
+func (m *chatMessage) IsStringContent() bool {
+	_, ok := m.Content.(string)
+	return ok
+}
+
+func (m *chatMessage) StringContent() string {
+	content, ok := m.Content.(string)
+	if ok {
+		return content
+	}
+	contentList, ok := m.Content.([]any)
+	if ok {
+		var contentStr string
+		for _, contentItem := range contentList {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
+			}
+			if contentMap["type"] == contentTypeText {
+				if subStr, ok := contentMap[contentTypeText].(string); ok {
+					contentStr += subStr + "\n"
+				}
+			}
+		}
+		return contentStr
+	}
+	return ""
+}
+
+func (m *chatMessage) ParseContent() []messageContent {
+	var contentList []messageContent
+	content, ok := m.Content.(string)
+	if ok {
+		contentList = append(contentList, messageContent{
+			Type: contentTypeText,
+			Text: content,
+		})
+		return contentList
+	}
+	anyList, ok := m.Content.([]any)
+	if ok {
+		for _, contentItem := range anyList {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
+			}
+			switch contentMap["type"] {
+			case contentTypeText:
+				if subStr, ok := contentMap[contentTypeText].(string); ok {
+					contentList = append(contentList, messageContent{
+						Type: contentTypeText,
+						Text: subStr,
+					})
+				}
+			case contentTypeImageUrl:
+				if subObj, ok := contentMap[contentTypeImageUrl].(map[string]any); ok {
+					contentList = append(contentList, messageContent{
+						Type: contentTypeImageUrl,
+						ImageUrl: &imageUrl{
+							Url: subObj["url"].(string),
+						},
+					})
+				}
+			}
+		}
+		return contentList
+	}
+	return nil
+}
+
 type toolCall struct {
 	Index    int          `json:"index"`
 	Id       string       `json:"id"`

plugins/wasm-go/extensions/ai-proxy/provider/qwen.go

@@ -22,17 +22,19 @@ import (
 const (
 	qwenResultFormatMessage = "message"
 
-	qwenDomain             = "dashscope.aliyuncs.com"
+	qwenDomain                   = "dashscope.aliyuncs.com"
-	qwenChatCompletionPath = "/api/v1/services/aigc/text-generation/generation"
+	qwenChatCompletionPath       = "/api/v1/services/aigc/text-generation/generation"
-	qwenTextEmbeddingPath  = "/api/v1/services/embeddings/text-embedding/text-embedding"
+	qwenTextEmbeddingPath        = "/api/v1/services/embeddings/text-embedding/text-embedding"
-	qwenCompatiblePath     = "/compatible-mode/v1/chat/completions"
+	qwenCompatiblePath           = "/compatible-mode/v1/chat/completions"
+	qwenMultimodalGenerationPath = "/api/v1/services/aigc/multimodal-generation/generation"
 
 	qwenTopPMin = 0.000001
 	qwenTopPMax = 0.999999
 
 	qwenDummySystemMessageContent = "You are a helpful assistant."
 
-	qwenLongModelName = "qwen-long"
+	qwenLongModelName     = "qwen-long"
+	qwenVlModelPrefixName = "qwen-vl"
 )
 
 type qwenProviderInitializer struct {
@@ -163,6 +165,10 @@ func (m *qwenProvider) onChatCompletionRequestBody(ctx wrapper.HttpContext, body
 	}
 	request.Model = mappedModel
 	ctx.SetContext(ctxKeyFinalRequestModel, request.Model)
+	// Use the qwen multimodal model generation API
+	if strings.HasPrefix(request.Model, qwenVlModelPrefixName) {
+		_ = util.OverwriteRequestPath(qwenMultimodalGenerationPath)
+	}
 
 	streaming := request.Stream
 	if streaming {
@@ -450,8 +456,29 @@ func (m *qwenProvider) buildChatCompletionStreamingResponse(ctx wrapper.HttpCont
 		if pushedMessage, ok := ctx.GetContext(ctxKeyPushedMessage).(qwenMessage); ok {
 			if message.Content == "" {
 				message.Content = pushedMessage.Content
+			} else if message.IsStringContent() {
+				deltaContentMessage.Content = util.StripPrefix(deltaContentMessage.StringContent(), pushedMessage.StringContent())
+			} else if strings.HasPrefix(baseMessage.Model, qwenVlModelPrefixName) {
+				// Use the Qwen multimodal model generation API
+				deltaContentList, ok := deltaContentMessage.Content.([]qwenVlMessageContent)
+				if !ok {
+					log.Warnf("unexpected deltaContentMessage content type: %T", deltaContentMessage.Content)
+				} else {
+					pushedContentList, ok := pushedMessage.Content.([]qwenVlMessageContent)
+					if !ok {
+						log.Warnf("unexpected pushedMessage content type: %T", pushedMessage.Content)
+					} else {
+						for i, content := range deltaContentList {
+							if i >= len(pushedContentList) {
+								break
+							}
+							pushedText := pushedContentList[i].Text
+							content.Text = util.StripPrefix(content.Text, pushedText)
+							deltaContentList[i] = content
+						}
+					}
+				}
 			}
-			deltaContentMessage.Content = util.StripPrefix(deltaContentMessage.Content, pushedMessage.Content)
 			if len(deltaToolCallsMessage.ToolCalls) > 0 && pushedMessage.ToolCalls != nil {
 				for i, tc := range deltaToolCallsMessage.ToolCalls {
 					if i >= len(pushedMessage.ToolCalls) {
@@ -557,7 +584,7 @@ func (m *qwenProvider) insertContextMessage(request *qwenTextGenRequest, content
 			if builder.Len() != 0 {
 				builder.WriteString("\n")
 			}
-			builder.WriteString(message.Content)
+			builder.WriteString(message.StringContent())
 		}
 		request.Input.Messages = append([]qwenMessage{{Role: roleSystem, Content: builder.String()}, fileMessage}, request.Input.Messages[firstNonSystemMessageIndex:]...)
 		return 1
@@ -662,10 +689,15 @@ type qwenUsage struct {
 type qwenMessage struct {
 	Name      string     `json:"name,omitempty"`
 	Role      string     `json:"role"`
-	Content   string     `json:"content"`
+	Content   any        `json:"content"`
 	ToolCalls []toolCall `json:"tool_calls,omitempty"`
 }
 
+type qwenVlMessageContent struct {
+	Image string `json:"image,omitempty"`
+	Text  string `json:"text,omitempty"`
+}
+
 type qwenTextEmbeddingRequest struct {
 	Model      string                      `json:"model"`
 	Input      qwenTextEmbeddingInput      `json:"input"`
@@ -705,11 +737,58 @@ func qwenMessageToChatMessage(qwenMessage qwenMessage) chatMessage {
 	}
 }
 
+func (m *qwenMessage) IsStringContent() bool {
+	_, ok := m.Content.(string)
+	return ok
+}
+
+func (m *qwenMessage) StringContent() string {
+	content, ok := m.Content.(string)
+	if ok {
+		return content
+	}
+	contentList, ok := m.Content.([]any)
+	if ok {
+		var contentStr string
+		for _, contentItem := range contentList {
+			contentMap, ok := contentItem.(map[string]any)
+			if !ok {
+				continue
+			}
+			if text, ok := contentMap["text"].(string); ok {
+				contentStr += text
+			}
+		}
+		return contentStr
+	}
+	return ""
+}
+
 func chatMessage2QwenMessage(chatMessage chatMessage) qwenMessage {
-	return qwenMessage{
+	if chatMessage.IsStringContent() {
-		Name:      chatMessage.Name,
+		return qwenMessage{
-		Role:      chatMessage.Role,
+			Name:      chatMessage.Name,
-		Content:   chatMessage.Content,
+			Role:      chatMessage.Role,
-		ToolCalls: chatMessage.ToolCalls,
+			Content:   chatMessage.StringContent(),
+			ToolCalls: chatMessage.ToolCalls,
+		}
+	} else {
+		var contents []qwenVlMessageContent
+		openaiContent := chatMessage.ParseContent()
+		for _, part := range openaiContent {
+			var content qwenVlMessageContent
+			if part.Type == contentTypeText {
+				content.Text = part.Text
+			} else if part.Type == contentTypeImageUrl {
+				content.Image = part.ImageUrl.Url
+			}
+			contents = append(contents, content)
+		}
+		return qwenMessage{
+			Name:      chatMessage.Name,
+			Role:      chatMessage.Role,
+			Content:   contents,
+			ToolCalls: chatMessage.ToolCalls,
+		}
 	}
 }

plugins/wasm-go/extensions/cluster-key-rate-limit/config.go

@@ -3,11 +3,12 @@ package main
 import (
 	"errors"
 	"fmt"
+	"strings"
+
 	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
 	"github.com/tidwall/gjson"
 	re "github.com/wasilibs/go-re2"
 	"github.com/zmap/go-iptree/iptree"
-	"strings"
 )
 
 // 限流规则项类型

plugins/wasm-go/extensions/cluster-key-rate-limit/utils.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"fmt"
-	"github.com/zmap/go-iptree/iptree"
 	"sort"
 	"strings"
+
+	"github.com/zmap/go-iptree/iptree"
 )
 
 // parseIPNet 解析Ip段配置

plugins/wasm-go/extensions/ext-auth/config.go

@@ -2,12 +2,13 @@ package main
 
 import (
 	"errors"
-	"ext-auth/expr"
 	"fmt"
-	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
-	"github.com/tidwall/gjson"
 	"net/http"
 	"strings"
+
+	"ext-auth/expr"
+	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
+	"github.com/tidwall/gjson"
 )
 
 const (

plugins/wasm-go/extensions/ext-auth/expr/matcher.go

@@ -2,9 +2,10 @@ package expr
 
 import (
 	"errors"
+	"strings"
+
 	"github.com/tidwall/gjson"
 	regexp "github.com/wasilibs/go-re2"
-	"strings"
 )
 
 const (

plugins/wasm-go/extensions/ext-auth/expr/matcher_test.go

@@ -1,9 +1,10 @@
 package expr
 
 import (
+	"testing"
+
 	"github.com/stretchr/testify/assert"
 	"github.com/tidwall/gjson"
-	"testing"
 )
 
 func TestStringMatcher(t *testing.T) {

plugins/wasm-go/extensions/ext-auth/main.go

@@ -15,11 +15,12 @@
 package main
 
 import (
+	"net/http"
+	"net/url"
+
 	"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
 	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
 	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
-	"net/http"
-	"net/url"
 )
 
 func main() {

plugins/wasm-go/extensions/ext-auth/utils.go

@@ -1,10 +1,11 @@
 package main
 
 import (
-	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
 	"net/http"
 	"sort"
 	"strings"
+
+	"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
 )
 
 func sendResponse(statusCode uint32, statusCodeDetailData string, headers http.Header) error {