jiazhizhong/higress
1
0
Fork 0
mirror of https://github.com/alibaba/higress.git synced 2026-06-04 18:17:33 +08:00
Code Issues Packages Projects Releases Wiki Activity

ai-proxy: support custom openai provider (#1176)

Browse Source 
Co-authored-by: Kent Dong <ch3cho@qq.com>
This commit is contained in:
澄潭
2024-08-07 10:33:01 +08:00
committed by GitHub
parent 536a3069a8
commit 0a578c2a04
17 changed files with 110 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
plugins/wasm-go/extensions/ai-proxy/README.md
View File

@@ -19,14 +19,14 @@ description: AI 代理插件配置参考
`provider`的配置字段说明如下：
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
| -------------- | --------------- | -------- | ------ |-------------------------------------------------------------------------------------------------------------------------------------------------------------|
| `type` | string | 必填 | - | AI 服务提供商名称 |
| `apiTokens` | array of string | 必填 | - | 用于在访问 AI 服务时进行认证的令牌。如果配置了多个 token，插件会在请求时随机进行选择。部分服务提供商只支持配置一个 token。 |
| `timeout` | number | 非必填 | - | 访问 AI 服务的超时时间。单位为毫秒。默认值为 120000，即 2 分钟 |
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
| -------------- | --------------- | -------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `type` | string | 必填 | - | AI 服务提供商名称 |
| `apiTokens` | array of string | 非必填 | - | 用于在访问 AI 服务时进行认证的令牌。如果配置了多个 token，插件会在请求时随机进行选择。部分服务提供商只支持配置一个 token。 |
| `timeout` | number | 非必填 | - | 访问 AI 服务的超时时间。单位为毫秒。默认值为 120000，即 2 分钟 |
| `modelMapping` | map of string | 非必填 | - | AI 模型映射表，用于将请求中的模型名称映射为服务提供商支持模型名称。<br/>1. 支持前缀匹配。例如用 "gpt-3-*" 匹配所有名称以“gpt-3-”开头的模型；<br/>2. 支持使用 "*" 为键来配置通用兜底映射关系；<br/>3. 如果映射的目标名称为空字符串 ""，则表示保留原模型名称。 |
| `protocol` | string | 非必填 | - | 插件对外提供的 API 接口契约。目前支持以下取值：openai（默认值，使用 OpenAI 的接口契约）、original（使用目标服务提供商的原始接口契约） |
| `context` | object | 非必填 | - | 配置 AI 对话上下文信息 |
| `protocol` | string | 非必填 | - | 插件对外提供的 API 接口契约。目前支持以下取值：openai（默认值，使用 OpenAI 的接口契约）、original（使用目标服务提供商的原始接口契约） |
| `context` | object | 非必填 | - | 配置 AI 对话上下文信息 |
`context`的配置字段说明如下：
@@ -40,7 +40,12 @@ description: AI 代理插件配置参考
#### OpenAI
OpenAI 所对应的 `type` 为 `openai`。它并无特有的配置字段。
OpenAI 所对应的 `type` 为 `openai`。它特有的配置字段如下:
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
|-------------------|----------|----------|--------|-------------------------------------------------------------------------------|
| `openaiCustomUrl` | string | 非必填 | - | 基于OpenAI协议的自定义后端URL，例如: www.example.com/myai/v1/chat/completions |
#### Azure OpenAI

12
plugins/wasm-go/extensions/ai-proxy/provider/azure.go
View File

@@ -23,6 +23,9 @@ func (m *azureProviderInitializer) ValidateConfig(config ProviderConfig) error {
if _, err := url.Parse(config.azureServiceUrl); err != nil {
return fmt.Errorf("invalid azureServiceUrl: %w", err)
}
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}
@@ -66,13 +69,16 @@ func (m *azureProvider) OnRequestBody(ctx wrapper.HttpContext, apiName ApiName,
if apiName != ApiNameChatCompletion {
return types.ActionContinue, errUnsupportedApiName
}
if m.contextCache == nil {
return types.ActionContinue, nil
}
request := &chatCompletionRequest{}
if err := decodeChatCompletionRequest(body, request); err != nil {
return types.ActionContinue, err
}
if m.contextCache == nil {
if err := replaceJsonRequestBody(request, log); err != nil {
_ = util.SendResponse(500, "ai-proxy.openai.set_include_usage_failed", util.MimeTypeTextPlain, fmt.Sprintf("failed to replace request body: %v", err))
}
return types.ActionContinue, nil
}
err := m.contextCache.GetContent(func(content string, err error) {
defer func() {
_ = proxywasm.ResumeHttpRequest()

4
plugins/wasm-go/extensions/ai-proxy/provider/baichuan.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -20,6 +21,9 @@ type baichuanProviderInitializer struct {
}
func (m *baichuanProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

3
plugins/wasm-go/extensions/ai-proxy/provider/baidu.go
View File

@@ -34,6 +34,9 @@ type baiduProviderInitializer struct {
}
func (b *baiduProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

8
plugins/wasm-go/extensions/ai-proxy/provider/claude.go
View File

@@ -4,12 +4,13 @@ import (
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
"strings"
"time"
)
// claudeProvider is the provider for Claude service.
@@ -78,6 +79,9 @@ type claudeTextGenDelta struct {
}
func (c *claudeProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

3
plugins/wasm-go/extensions/ai-proxy/provider/cloudflare.go
View File

@@ -21,6 +21,9 @@ type cloudflareProviderInitializer struct {
}
func (c *cloudflareProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

4
plugins/wasm-go/extensions/ai-proxy/provider/deepseek.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -20,6 +21,9 @@ type deepseekProviderInitializer struct {
}
func (m *deepseekProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

4
plugins/wasm-go/extensions/ai-proxy/provider/groq.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -18,6 +19,9 @@ const (
type groqProviderInitializer struct{}
func (m *groqProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

3
plugins/wasm-go/extensions/ai-proxy/provider/minimax.go
View File

@@ -52,6 +52,9 @@ func (m *minimaxProviderInitializer) ValidateConfig(config ProviderConfig) error
}
}
}
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

3
plugins/wasm-go/extensions/ai-proxy/provider/moonshot.go
View File

@@ -26,6 +26,9 @@ func (m *moonshotProviderInitializer) ValidateConfig(config ProviderConfig) erro
if config.moonshotFileId != "" && config.context != nil {
return errors.New("moonshotFileId and context cannot be configured at the same time")
}
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

58
plugins/wasm-go/extensions/ai-proxy/provider/openai.go
View File

@@ -2,6 +2,7 @@ package provider
import (
"fmt"
"strings"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
@@ -12,9 +13,9 @@ import (
// openaiProvider is the provider for OpenAI service.
const (
openaiDomain = "api.openai.com"
openaiChatCompletionPath = "/v1/chat/completions"
openaiEmbeddingsPath = "/v1/chat/embeddings"
defaultOpenaiDomain = "api.openai.com"
defaultOpenaiChatCompletionPath = "/v1/chat/completions"
defaultOpenaiEmbeddingsPath = "/v1/chat/embeddings"
)
type openaiProviderInitializer struct {
@@ -25,14 +26,29 @@ func (m *openaiProviderInitializer) ValidateConfig(config ProviderConfig) error
}
func (m *openaiProviderInitializer) CreateProvider(config ProviderConfig) (Provider, error) {
if config.openaiCustomUrl == "" {
return &openaiProvider{
config: config,
contextCache: createContextCache(&config),
}, nil
}
customUrl := strings.TrimPrefix(strings.TrimPrefix(config.openaiCustomUrl, "http://"), "https://")
pairs := strings.SplitN(customUrl, "/", 2)
if len(pairs) != 2 {
return nil, fmt.Errorf("invalid openaiCustomUrl:%s", config.openaiCustomUrl)
}
return &openaiProvider{
config: config,
customDomain: pairs[0],
customPath: "/" + pairs[1],
contextCache: createContextCache(&config),
}, nil
}
type openaiProvider struct {
config ProviderConfig
customDomain string
customPath string
contextCache *contextCache
}
@@ -41,15 +57,24 @@ func (m *openaiProvider) GetProviderType() string {
}
func (m *openaiProvider) OnRequestHeaders(ctx wrapper.HttpContext, apiName ApiName, log wrapper.Log) (types.Action, error) {
switch apiName {
case ApiNameChatCompletion:
_ = util.OverwriteRequestPath(openaiChatCompletionPath)
break
case ApiNameEmbeddings:
_ = util.OverwriteRequestPath(openaiEmbeddingsPath)
break
if m.customPath == "" {
switch apiName {
case ApiNameChatCompletion:
_ = util.OverwriteRequestPath(defaultOpenaiChatCompletionPath)
case ApiNameEmbeddings:
_ = util.OverwriteRequestPath(defaultOpenaiEmbeddingsPath)
}
} else {
_ = util.OverwriteRequestPath(m.customPath)
}
if m.customDomain == "" {
_ = util.OverwriteRequestHost(defaultOpenaiDomain)
} else {
_ = util.OverwriteRequestHost(m.customDomain)
}
if len(m.config.apiTokens) > 0 {
_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetRandomToken())
}
_ = util.OverwriteRequestAuthorization("Bearer " + m.config.GetRandomToken())
_ = proxywasm.RemoveHttpRequestHeader("Content-Length")
return types.ActionContinue, nil
}
@@ -63,26 +88,19 @@ func (m *openaiProvider) OnRequestBody(ctx wrapper.HttpContext, apiName ApiName,
if err := decodeChatCompletionRequest(body, request); err != nil {
return types.ActionContinue, err
}
bodyAltered := false
if request.Stream {
// For stream requests, we need to include usage in the response.
if request.StreamOptions == nil {
request.StreamOptions = &streamOptions{IncludeUsage: true}
bodyAltered = true
} else if !request.StreamOptions.IncludeUsage {
request.StreamOptions.IncludeUsage = true
bodyAltered = true
}
}
if m.contextCache == nil {
if bodyAltered {
if err := replaceJsonRequestBody(request, log); err != nil {
_ = util.SendResponse(500, "ai-proxy.openai.set_include_usage_failed", util.MimeTypeTextPlain, fmt.Sprintf("failed to replace request body: %v", err))
}
if err := replaceJsonRequestBody(request, log); err != nil {
_ = util.SendResponse(500, "ai-proxy.openai.set_include_usage_failed", util.MimeTypeTextPlain, fmt.Sprintf("failed to replace request body: %v", err))
}
return types.ActionContinue, nil
} else {
// If context cache is configured and body has been altered, the new body will be replaced when inserting the context data.
}
err := m.contextCache.GetContent(func(content string, err error) {
defer func() {

7
plugins/wasm-go/extensions/ai-proxy/provider/provider.go
View File

@@ -121,6 +121,9 @@ type ProviderConfig struct {
// @Title zh-CN 请求超时
// @Description zh-CN 请求AI服务的超时时间，单位为毫秒。默认值为120000，即2分钟
timeout uint32 `required:"false" yaml:"timeout" json:"timeout"`
// @Title zh-CN 基于OpenAI协议的自定义后端URL
// @Description zh-CN 仅适用于支持 openai 协议的服务。
openaiCustomUrl string `required:"false" yaml:"openaiCustomUrl" json:"openaiCustomUrl"`
// @Title zh-CN Moonshot File ID
// @Description zh-CN 仅适用于Moonshot AI服务。Moonshot AI服务的文件ID，其内容用于补充AI请求上下文
moonshotFileId string `required:"false" yaml:"moonshotFileId" json:"moonshotFileId"`
@@ -175,6 +178,7 @@ func (c *ProviderConfig) FromJson(json gjson.Result) {
if c.timeout == 0 {
c.timeout = defaultTimeout
}
c.openaiCustomUrl = json.Get("openaiCustomUrl").String()
c.moonshotFileId = json.Get("moonshotFileId").String()
c.azureServiceUrl = json.Get("azureServiceUrl").String()
c.qwenFileIds = make([]string, 0)
@@ -205,9 +209,6 @@ func (c *ProviderConfig) FromJson(json gjson.Result) {
}
func (c *ProviderConfig) Validate() error {
if c.apiTokens == nil || len(c.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
if c.timeout < 0 {
return errors.New("invalid timeout in config")
}

3
plugins/wasm-go/extensions/ai-proxy/provider/qwen.go
View File

@@ -39,6 +39,9 @@ func (m *qwenProviderInitializer) ValidateConfig(config ProviderConfig) error {
if len(config.qwenFileIds) != 0 && config.context != nil {
return errors.New("qwenFileIds and context cannot be configured at the same time")
}
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

3
plugins/wasm-go/extensions/ai-proxy/provider/request_helper.go
View File

@@ -2,7 +2,6 @@ package provider
import (
"encoding/json"
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/pkg/wrapper"
@@ -14,7 +13,7 @@ func decodeChatCompletionRequest(body []byte, request *chatCompletionRequest) er
return fmt.Errorf("unable to unmarshal request: %v", err)
}
if request.Messages == nil || len(request.Messages) == 0 {
return errors.New("no message found in the request body")
return fmt.Errorf("no message found in the request body: %s", body)
}
return nil
}

4
plugins/wasm-go/extensions/ai-proxy/provider/stepfun.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -18,6 +19,9 @@ type stepfunProviderInitializer struct {
}
func (m *stepfunProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

4
plugins/wasm-go/extensions/ai-proxy/provider/yi.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -18,6 +19,9 @@ type yiProviderInitializer struct {
}
func (m *yiProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}

4
plugins/wasm-go/extensions/ai-proxy/provider/zhipuai.go
View File

@@ -1,6 +1,7 @@
package provider
import (
"errors"
"fmt"
"github.com/alibaba/higress/plugins/wasm-go/extensions/ai-proxy/util"
@@ -17,6 +18,9 @@ const (
type zhipuAiProviderInitializer struct{}
func (m *zhipuAiProviderInitializer) ValidateConfig(config ProviderConfig) error {
if config.apiTokens == nil || len(config.apiTokens) == 0 {
return errors.New("no apiToken found in provider config")
}
return nil
}