feat(ai-proxy): 添加 Claude 图片理解与 Tools 调用能力 || feat(ai-proxy): Add Claude image understanding and Tools calling capabilities (#2385)

Signed-off-by: Xijun Dai <daixijun1990@gmail.com>

plugins/wasm-go/extensions/ai-proxy/provider/model.go

@@ -20,8 +20,10 @@ const (
 
 	httpStatus200 = "200"
 
-	contentTypeText     = "text"
-	contentTypeImageUrl = "image_url"
+	contentTypeText       = "text"
+	contentTypeImageUrl   = "image_url"
+	contentTypeInputAudio = "input_audio"
+	contentTypeFile       = "file"
 
 	reasoningStartTag = "<think>"
 	reasoningEndTag   = "</think>"
@@ -53,11 +55,40 @@ type chatCompletionRequest struct {
 	Temperature         float64                `json:"temperature,omitempty"`
 	TopP                float64                `json:"top_p,omitempty"`
 	Tools               []tool                 `json:"tools,omitempty"`
-	ToolChoice          *toolChoice            `json:"tool_choice,omitempty"`
+	ToolChoice          interface{}            `json:"tool_choice,omitempty"`
 	ParallelToolCalls   bool                   `json:"parallel_tool_calls,omitempty"`
 	User                string                 `json:"user,omitempty"`
 }
 
+func (c *chatCompletionRequest) getMaxTokens() int {
+	if c.MaxCompletionTokens > 0 {
+		return c.MaxCompletionTokens
+	}
+	return c.MaxTokens
+}
+
+func (c *chatCompletionRequest) getToolChoiceString() string {
+	if c.ToolChoice == nil {
+		return ""
+	}
+
+	if tc, ok := c.ToolChoice.(string); ok {
+		return tc
+	}
+	return ""
+}
+
+func (c *chatCompletionRequest) getToolChoiceObject() *toolChoice {
+	if c.ToolChoice == nil {
+		return nil
+	}
+
+	if tc, ok := c.ToolChoice.(*toolChoice); ok {
+		return tc
+	}
+	return nil
+}
+
 type CompletionRequest struct {
 	Model            string         `json:"model"`
 	Prompt           string         `json:"prompt"`
@@ -200,13 +231,26 @@ func (m *chatMessage) handleStreamingReasoningContent(ctx wrapper.HttpContext, r
 	}
 }
 
-type messageContent struct {
-	Type     string    `json:"type,omitempty"`
-	Text     string    `json:"text"`
-	ImageUrl *imageUrl `json:"image_url,omitempty"`
+type chatMessageContent struct {
+	Type       string                      `json:"type,omitempty"`
+	Text       string                      `json:"text"`
+	ImageUrl   *chatMessageContentImageUrl `json:"image_url,omitempty"`
+	File       *chatMessageContentFile     `json:"file,omitempty"`
+	InputAudio *chatMessageContentAudio    `json:"input_audio,omitempty"`
 }
 
-type imageUrl struct {
+type chatMessageContentAudio struct {
+	Data   string `json:"data"`
+	Format string `json:"format"`
+}
+
+type chatMessageContentFile struct {
+	FileData string `json:"file_data,omitempty"`
+	FileId   string `json:"file_id,omitempty"`
+	FileName string `json:"file_name,omitempty"`
+}
+
+type chatMessageContentImageUrl struct {
 	Url    string `json:"url,omitempty"`
 	Detail string `json:"detail,omitempty"`
 }
@@ -266,11 +310,11 @@ func (m *chatMessage) StringContent() string {
 	return ""
 }
 
-func (m *chatMessage) ParseContent() []messageContent {
-	var contentList []messageContent
+func (m *chatMessage) ParseContent() []chatMessageContent {
+	var contentList []chatMessageContent
 	content, ok := m.Content.(string)
 	if ok {
-		contentList = append(contentList, messageContent{
+		contentList = append(contentList, chatMessageContent{
 			Type: contentTypeText,
 			Text: content,
 		})
@@ -286,18 +330,43 @@ func (m *chatMessage) ParseContent() []messageContent {
 			switch contentMap["type"] {
 			case contentTypeText:
 				if subStr, ok := contentMap[contentTypeText].(string); ok {
-					contentList = append(contentList, messageContent{
+					contentList = append(contentList, chatMessageContent{
 						Type: contentTypeText,
 						Text: subStr,
 					})
 				}
 			case contentTypeImageUrl:
 				if subObj, ok := contentMap[contentTypeImageUrl].(map[string]any); ok {
-					contentList = append(contentList, messageContent{
+					msg := chatMessageContent{
 						Type: contentTypeImageUrl,
-						ImageUrl: &imageUrl{
+						ImageUrl: &chatMessageContentImageUrl{
 							Url: subObj["url"].(string),
 						},
+					}
+					if detail, ok := subObj["detail"].(string); ok {
+						msg.ImageUrl.Detail = detail
+					}
+					contentList = append(contentList, msg)
+				}
+			case contentTypeInputAudio:
+				if subObj, ok := contentMap[contentTypeInputAudio].(map[string]any); ok {
+					contentList = append(contentList, chatMessageContent{
+						Type: contentTypeInputAudio,
+						InputAudio: &chatMessageContentAudio{
+							Data:   subObj["data"].(string),
+							Format: subObj["format"].(string),
+						},
+					})
+				}
+			case contentTypeFile:
+				if subObj, ok := contentMap[contentTypeFile].(map[string]any); ok {
+					contentList = append(contentList, chatMessageContent{
+						Type: contentTypeFile,
+						File: &chatMessageContentFile{
+							FileId: subObj["file_id"].(string),
+							// FileName: subObj["file_name"].(string),
+							// FileData: subObj["file_data"].(string),
+						},
 					})
 				}
 			}