jiazhizhong/higress
1
0
Fork 0
mirror of https://github.com/alibaba/higress.git synced 2026-05-27 06:07:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

ai cache plugin support set skip ai cache header (#1380)

Browse Source
This commit is contained in:
澄潭
2024-10-21 15:43:01 +08:00
committed by GitHub
parent fc6902ded2
commit badf4b7101
3 changed files with 21 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
plugins/wasm-go/extensions/ai-cache/README.md
View File

@@ -9,6 +9,11 @@ description: AI 缓存插件配置参考
LLM 结果缓存插件，默认配置方式可以直接用于 openai 协议的结果缓存，同时支持流式和非流式响应的缓存。 LLM 结果缓存插件，默认配置方式可以直接用于 openai 协议的结果缓存，同时支持流式和非流式响应的缓存。
**提示**
携带请求头`x-higress-skip-ai-cache: on`时，当前请求将不会使用缓存中的内容，而是直接转发给后端服务，同时也不会缓存该请求返回响应的内容
## 运行属性 ## 运行属性
插件执行阶段：`认证阶段` 插件执行阶段：`认证阶段`

4
plugins/wasm-go/extensions/ai-cache/README_EN.md
View File

@@ -6,6 +6,10 @@ description: AI Cache Plugin Configuration Reference
## Function Description ## Function Description
LLM result caching plugin, the default configuration can be directly used for result caching under the OpenAI protocol, and it supports caching of both streaming and non-streaming responses. LLM result caching plugin, the default configuration can be directly used for result caching under the OpenAI protocol, and it supports caching of both streaming and non-streaming responses.
**Tips**
When carrying the request header `x-higress-skip-ai-cache: on`, the current request will not use content from the cache but will be directly forwarded to the backend service. Additionally, the response content from this request will not be cached.
## Runtime Properties ## Runtime Properties
Plugin Execution Phase: `Authentication Phase` Plugin Execution Phase: `Authentication Phase`
Plugin Execution Priority: `10` Plugin Execution Priority: `10`

12
plugins/wasm-go/extensions/ai-cache/main.go
View File

@@ -22,6 +22,7 @@ const (
ToolCallsContextKey = "toolCalls" ToolCallsContextKey = "toolCalls"
StreamContextKey = "stream" StreamContextKey = "stream"
DefaultCacheKeyPrefix = "higress-ai-cache:" DefaultCacheKeyPrefix = "higress-ai-cache:"
SkipCacheHeader = "x-higress-skip-ai-cache"
) )
func main() { func main() {
@@ -172,6 +173,12 @@ func parseConfig(json gjson.Result, c *PluginConfig, log wrapper.Log) error {
} }
func onHttpRequestHeaders(ctx wrapper.HttpContext, config PluginConfig, log wrapper.Log) types.Action { func onHttpRequestHeaders(ctx wrapper.HttpContext, config PluginConfig, log wrapper.Log) types.Action {
skipCache, _ := proxywasm.GetHttpRequestHeader(SkipCacheHeader)
if skipCache == "on" {
ctx.SetContext(SkipCacheHeader, struct{}{})
ctx.DontReadRequestBody()
return types.ActionContinue
}
contentType, _ := proxywasm.GetHttpRequestHeader("content-type") contentType, _ := proxywasm.GetHttpRequestHeader("content-type")
// The request does not have a body. // The request does not have a body.
if contentType == "" { if contentType == "" {
@@ -270,6 +277,11 @@ func processSSEMessage(ctx wrapper.HttpContext, config PluginConfig, sseMessage
} }
func onHttpResponseHeaders(ctx wrapper.HttpContext, config PluginConfig, log wrapper.Log) types.Action { func onHttpResponseHeaders(ctx wrapper.HttpContext, config PluginConfig, log wrapper.Log) types.Action {
skipCache := ctx.GetContext(SkipCacheHeader)
if skipCache != nil {
ctx.DontReadResponseBody()
return types.ActionContinue
}
contentType, _ := proxywasm.GetHttpResponseHeader("content-type") contentType, _ := proxywasm.GetHttpResponseHeader("content-type")
if strings.Contains(contentType, "text/event-stream") { if strings.Contains(contentType, "text/event-stream") {
ctx.SetContext(StreamContextKey, struct{}{}) ctx.SetContext(StreamContextKey, struct{}{})