jiazhizhong/higress
1
0
Fork 0
mirror of https://github.com/alibaba/higress.git synced 2026-02-06 23:21:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: support retry on http status code (#1817)

Browse Source 
Co-authored-by: Kent Dong <ch3cho@qq.com>
This commit is contained in:
Se7en
2025-03-11 13:38:02 +08:00
committed by GitHub
parent beb60fcacd
commit 19496e5759
7 changed files with 72 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
plugins/wasm-go/extensions/ai-proxy/README.md
View File

@@ -81,22 +81,27 @@ custom-setting会遵循如下表格，根据`name`和协议来替换对应的字
`failover` 的配置字段说明如下：
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
|------------------|--------|-----------------|-------|-----------------------------|
| enabled | bool | 非必填 | false | 是否启用 apiToken 的 failover 机制 |
| failureThreshold | int | 非必填 | 3 | 触发 failover 连续请求失败的阈值（次数） |
| successThreshold | int | 非必填 | 1 | 健康检测的成功阈值（次数） |
| healthCheckInterval | int | 非必填 | 5000 | 健康检测的间隔时间，单位毫秒 |
| healthCheckTimeout | int | 非必填 | 5000 | 健康检测的超时时间，单位毫秒 |
| healthCheckModel | string | 启用 failover 时必填 | | 健康检测使用的模型 |
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
|------------------|--------|-----------------|-------|-----------------------------------|
| enabled | bool | 非必填 | false | 是否启用 apiToken 的 failover 机制 |
| failureThreshold | int | 非必填 | 3 | 触发 failover 连续请求失败的阈值（次数） |
| successThreshold | int | 非必填 | 1 | 健康检测的成功阈值（次数） |
| healthCheckInterval | int | 非必填 | 5000 | 健康检测的间隔时间，单位毫秒 |
| healthCheckTimeout | int | 非必填 | 5000 | 健康检测的超时时间，单位毫秒 |
| healthCheckModel | string | 启用 failover 时必填 | | 健康检测使用的模型 |
| failoverOnStatus | array of string | 非必填 | ["4.*", "5.*"] | 需要进行 failover 的原始请求的状态码，支持正则表达式匹配 |
`retryOnFailure` 的配置字段说明如下：
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
|------------------|--------|-----------------|-------|-------------|
| enabled | bool | 非必填 | false | 是否启用失败请求重试 |
| maxRetries | int | 非必填 | 1 | 最大重试次数 |
| retryTimeout | int | 非必填 | 30000 | 重试超时时间，单位毫秒 |
目前仅支持对非流式请求进行重试。
| 名称 | 数据类型 | 填写要求 | 默认值 | 描述 |
|------------------|--------|--------|-------|---------------------------|
| enabled | bool | 非必填 | false | 是否启用失败请求重试 |
| maxRetries | int | 非必填 | 1 | 最大重试次数 |
| retryTimeout | int | 非必填 | 30000 | 重试超时时间，单位毫秒 |
| retryOnStatus | array of string | 非必填 | ["4.*", "5.*"] | 需要进行重试的原始请求的状态码，支持正则表达式匹配 |
### 提供商特有配置

8
plugins/wasm-go/extensions/ai-proxy/go.mod
View File

@@ -13,11 +13,17 @@ require (
github.com/tidwall/gjson v1.17.3
)
require (
github.com/tetratelabs/wazero v1.7.2 // indirect
github.com/wasilibs/go-re2 v1.6.0 // indirect
golang.org/x/sys v0.21.0 // indirect
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/google/uuid v1.3.0
github.com/higress-group/nottinygc v0.0.0-20231101025119-e93c4c2f8520 // indirect
github.com/magefile/mage v1.14.0 // indirect
github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect

8
plugins/wasm-go/extensions/ai-proxy/go.sum
View File

@@ -8,10 +8,14 @@ github.com/higress-group/proxy-wasm-go-sdk v1.0.0 h1:BZRNf4R7jr9hwRivg/E29nkVaKE
github.com/higress-group/proxy-wasm-go-sdk v1.0.0/go.mod h1:iiSyFbo+rAtbtGt/bsefv8GU57h9CCLYGJA74/tF5/0=
github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho=
github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.17.3 h1:bwWLZU7icoKRG+C+0PNwIKC6FCJO/Q3p2pZvuP0jN94=
github.com/tidwall/gjson v1.17.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -23,6 +27,10 @@ github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE=
github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/wasilibs/go-re2 v1.6.0 h1:CLlhDebt38wtl/zz4ww+hkXBMcxjrKFvTDXzFW2VOz8=
github.com/wasilibs/go-re2 v1.6.0/go.mod h1:prArCyErsypRBI/jFAFJEbzyHzjABKqkzlidF0SNA04=
golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

2
plugins/wasm-go/extensions/ai-proxy/main.go
View File

@@ -189,7 +189,7 @@ func onHttpResponseHeaders(ctx wrapper.HttpContext, pluginConfig config.PluginCo
log.Errorf("unable to load :status header from response: %v", err)
}
ctx.DontReadResponseBody()
return providerConfig.OnRequestFailed(activeProvider, ctx, apiTokenInUse, apiTokens, log)
return providerConfig.OnRequestFailed(activeProvider, ctx, apiTokenInUse, apiTokens, status, log)
}
// Reset ctxApiTokenRequestFailureCount if the request is successful,

20
plugins/wasm-go/extensions/ai-proxy/provider/failover.go
View File

@@ -30,6 +30,8 @@ type failover struct {
healthCheckTimeout int64 `required:"false" yaml:"healthCheckTimeout" json:"healthCheckTimeout"`
// @Title zh-CN 健康检测使用的模型
healthCheckModel string `required:"false" yaml:"healthCheckModel" json:"healthCheckModel"`
// @Title zh-CN 需要进行 failover 的原始请求的状态码，支持正则表达式匹配
failoverOnStatus []string `required:"false" yaml:"failoverOnStatus" json:"failoverOnStatus"`
// @Title zh-CN 本次请求使用的 apiToken
ctxApiTokenInUse string
// @Title zh-CN 记录本次请求时所有可用的 apiToken
@@ -92,6 +94,14 @@ func (f *failover) FromJson(json gjson.Result) {
f.healthCheckTimeout = 5000
}
f.healthCheckModel = json.Get("healthCheckModel").String()
for _, status := range json.Get("failoverOnStatus").Array() {
f.failoverOnStatus = append(f.failoverOnStatus, status.String())
}
// If failoverOnStatus is empty, default to retry on 4xx and 5xx
if len(f.failoverOnStatus) == 0 {
f.failoverOnStatus = []string{"4.*", "5.*"}
}
}
func (f *failover) Validate() error {
@@ -558,17 +568,21 @@ func (c *ProviderConfig) resetSharedData() {
_ = proxywasm.SetSharedData(c.failover.ctxApiTokenRequestFailureCount, nil, 0)
}
func (c *ProviderConfig) OnRequestFailed(activeProvider Provider, ctx wrapper.HttpContext, apiTokenInUse string, apiTokens []string, log wrapper.Log) types.Action {
if c.isFailoverEnabled() {
func (c *ProviderConfig) OnRequestFailed(activeProvider Provider, ctx wrapper.HttpContext, apiTokenInUse string, apiTokens []string, status string, log wrapper.Log) types.Action {
if c.isFailoverEnabled() && util.MatchStatus(status, c.failover.failoverOnStatus) {
c.handleUnavailableApiToken(ctx, apiTokenInUse, log)
}
if c.isRetryOnFailureEnabled() && ctx.GetContext(ctxKeyIsStreaming) != nil && !ctx.GetContext(ctxKeyIsStreaming).(bool) {
if c.isRetryOnFailureEnabled() && util.MatchStatus(status, c.retryOnFailure.retryOnStatus) && isNotStreamingResponse(ctx) {
c.retryFailedRequest(activeProvider, ctx, apiTokenInUse, apiTokens, log)
return types.HeaderStopAllIterationAndWatermark
}
return types.ActionContinue
}
func isNotStreamingResponse(ctx wrapper.HttpContext) bool {
return ctx.GetContext(ctxKeyIsStreaming) != nil && !ctx.GetContext(ctxKeyIsStreaming).(bool)
}
func (c *ProviderConfig) GetApiTokenInUse(ctx wrapper.HttpContext) string {
token, _ := ctx.GetContext(c.failover.ctxApiTokenInUse).(string)
return token

9
plugins/wasm-go/extensions/ai-proxy/provider/retry.go
View File

@@ -22,6 +22,8 @@ type retryOnFailure struct {
maxRetries int64 `required:"false" yaml:"maxRetries" json:"maxRetries"`
// @Title zh-CN 重试超时时间
retryTimeout int64 `required:"false" yaml:"retryTimeout" json:"retryTimeout"`
// @Title zh-CN 需要进行重试的原始请求的状态码，支持正则表达式匹配
retryOnStatus []string `required:"false" yaml:"retryOnStatus" json:"retryOnStatus"`
}
func (r *retryOnFailure) FromJson(json gjson.Result) {
@@ -34,6 +36,13 @@ func (r *retryOnFailure) FromJson(json gjson.Result) {
if r.retryTimeout == 0 {
r.retryTimeout = 30 * 1000
}
for _, status := range json.Get("retryOnStatus").Array() {
r.retryOnStatus = append(r.retryOnStatus, status.String())
}
// If retryOnStatus is empty, default to retry on 4xx and 5xx
if len(r.retryOnStatus) == 0 {
r.retryOnStatus = []string{"4.*", "5.*"}
}
}
func (c *ProviderConfig) isRetryOnFailureEnabled() bool {

12
plugins/wasm-go/extensions/ai-proxy/util/string.go
View File

@@ -1,8 +1,20 @@
package util
import regexp "github.com/wasilibs/go-re2"
func StripPrefix(s string, prefix string) string {
if len(prefix) != 0 && len(s) >= len(prefix) && s[0:len(prefix)] == prefix {
return s[len(prefix):]
}
return s
}
func MatchStatus(status string, patterns []string) bool {
for _, pattern := range patterns {
matched, _ := regexp.MatchString(pattern, status)
if matched {
return true
}
}
return false
}