mirror of
https://github.com/alibaba/higress.git
synced 2026-07-28 07:10:23 +08:00
test: add #4034 conformance repro (redis-failure inject CPU spin)
Adds a wasm-go conformance case reproducing #4034: a deferred async redis-failure callback synchronously calls injectEncodedDataToFilterChain, which pre-fix makes WasmBase::doAfterVmCallActions re-queue an action forever and spins a worker at 100% CPU. Redis is pointed at an unroutable endpoint so the failure callbacks fire under concurrency. With the drain-to-local fix the gateway stays responsive (200); a spinning worker would fail via timeout. Validated end-to-end against the rebuilt gateway image. Signed-off-by: 澄潭 <zty98751@alibaba-inc.com>
This commit is contained in:
澄潭
@@ -0,0 +1 @@
|
||||
1
|
||||
20
plugins/wasm-go/extensions/test-redis-inject-spin/go.mod
Normal file
20
plugins/wasm-go/extensions/test-redis-inject-spin/go.mod
Normal file
@@ -0,0 +1,20 @@
|
||||
module github.com/alibaba/higress/plugins/wasm-go/extensions/test-redis-inject-spin
|
||||
|
||||
go 1.24.1
|
||||
|
||||
toolchain go1.24.4
|
||||
|
||||
require (
|
||||
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2
|
||||
github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d
|
||||
github.com/tidwall/gjson v1.18.0
|
||||
github.com/tidwall/resp v0.1.1
|
||||
google.golang.org/protobuf v1.36.6
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/tidwall/match v1.1.1 // indirect
|
||||
github.com/tidwall/pretty v1.2.1 // indirect
|
||||
github.com/tidwall/sjson v1.2.5 // indirect
|
||||
)
|
||||
32
plugins/wasm-go/extensions/test-redis-inject-spin/go.sum
Normal file
32
plugins/wasm-go/extensions/test-redis-inject-spin/go.sum
Normal file
@@ -0,0 +1,32 @@
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2 h1:NY33OrWCJJ+DFiLc+lsBY4Ywor2Ik61ssk6qkGF8Ypo=
|
||||
github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2/go.mod h1:tRI2LfMudSkKHhyv1uex3BWzcice2s/l8Ah8axporfA=
|
||||
github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d h1:LgYbzEBtg0+LEqoebQeMVgAB6H5SgqG+KN+gBhNfKbM=
|
||||
github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d/go.mod h1:uKVYICbRaxTlKqdm8E0dpjbysxM8uCPb9LV26hF3Km8=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
|
||||
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE=
|
||||
github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0=
|
||||
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
|
||||
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
||||
google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
119
plugins/wasm-go/extensions/test-redis-inject-spin/main.go
Normal file
119
plugins/wasm-go/extensions/test-redis-inject-spin/main.go
Normal file
@@ -0,0 +1,119 @@
|
||||
// Repro plugin for higress issue #4034: worker CPU spin in
|
||||
// WasmBase::doAfterVmCallActions when a deferred async callback synchronously
|
||||
// calls injectEncodedDataToFilterChain (nesting SaveRestoreContext) so that the
|
||||
// after-vm-call action re-queues itself forever.
|
||||
//
|
||||
// Repro shape (dead-Redis only, self-contained — no external HTTP dependency):
|
||||
// 1. In the response-header phase, pause the response.
|
||||
// 2. Fire N Redis commands against an UNREACHABLE redis cluster.
|
||||
// 3. Each failure callback (onRedisCallFailure) synchronously calls the
|
||||
// inject_encoded_data_to_filter_chain foreign function. Because sibling
|
||||
// failure callbacks are still queued as after-vm-call actions, the nested
|
||||
// SaveRestoreContext leaves current_context_ != nullptr and the deferred
|
||||
// action re-queues itself -> CPU spin on the pre-fix host.
|
||||
//
|
||||
// With the Layer A drain-to-local fix in place, the drain terminates over a
|
||||
// snapshot and worker CPU stays bounded.
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm"
|
||||
"github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types"
|
||||
"github.com/higress-group/wasm-go/pkg/log"
|
||||
pb "github.com/higress-group/wasm-go/pkg/protos"
|
||||
"github.com/higress-group/wasm-go/pkg/wrapper"
|
||||
"github.com/tidwall/gjson"
|
||||
"github.com/tidwall/resp"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
func main() {}
|
||||
|
||||
type Config struct {
|
||||
redisClient *wrapper.RedisClusterClient[wrapper.FQDNCluster]
|
||||
redisKey string
|
||||
injectBody string
|
||||
injectCount int
|
||||
}
|
||||
|
||||
func init() {
|
||||
wrapper.SetCtx(
|
||||
"test-redis-inject-spin",
|
||||
wrapper.ParseConfig(parseConfig),
|
||||
wrapper.ProcessResponseHeaders(onHttpResponseHeaders),
|
||||
)
|
||||
}
|
||||
|
||||
func parseConfig(json gjson.Result, config *Config) error {
|
||||
serviceName := json.Get("service_name").String()
|
||||
if serviceName == "" {
|
||||
serviceName = "dead-redis.dead-redis.svc.cluster.local"
|
||||
}
|
||||
servicePort := json.Get("service_port").Int()
|
||||
if servicePort == 0 {
|
||||
servicePort = 6379
|
||||
}
|
||||
config.redisKey = json.Get("redis_key").String()
|
||||
if config.redisKey == "" {
|
||||
config.redisKey = "higress-4034-key"
|
||||
}
|
||||
config.injectBody = json.Get("inject_body").String()
|
||||
if config.injectBody == "" {
|
||||
config.injectBody = "injected-by-4034-repro\n"
|
||||
}
|
||||
config.injectCount = int(json.Get("inject_count").Int())
|
||||
if config.injectCount == 0 {
|
||||
config.injectCount = 16
|
||||
}
|
||||
|
||||
config.redisClient = wrapper.NewRedisClusterClient(wrapper.FQDNCluster{
|
||||
FQDN: serviceName,
|
||||
Port: servicePort,
|
||||
})
|
||||
// Init never returns error for an unreachable host; commands' callbacks will
|
||||
// fire with an error value, which is exactly the #4034 trigger path.
|
||||
return config.redisClient.Init("", "", 1000)
|
||||
}
|
||||
|
||||
func onHttpResponseHeaders(ctx wrapper.HttpContext, config Config) types.Action {
|
||||
proxywasm.RemoveHttpResponseHeader("content-length")
|
||||
ctx.DontReadResponseBody()
|
||||
|
||||
inject := func() {
|
||||
d := &pb.InjectEncodedDataToFilterChainArguments{
|
||||
Body: config.injectBody,
|
||||
Endstream: true,
|
||||
}
|
||||
s, err := proto.Marshal(d)
|
||||
if err != nil {
|
||||
log.Errorf("marshal inject args failed: %+v", err)
|
||||
return
|
||||
}
|
||||
if _, err := proxywasm.CallForeignFunction("inject_encoded_data_to_filter_chain_on_header", s); err != nil {
|
||||
log.Errorf("call inject_encoded_data_to_filter_chain_on_header failed: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
scheduled := 0
|
||||
for i := 0; i < config.injectCount; i++ {
|
||||
err := config.redisClient.Get(config.redisKey, func(response resp.Value) {
|
||||
// Fires on redis-unreachable failure; synchronously inject to nest
|
||||
// SaveRestoreContext while sibling callbacks are still queued.
|
||||
if response.Error() != nil {
|
||||
log.Debugf("redis get failed as expected: %v", response.Error())
|
||||
}
|
||||
inject()
|
||||
})
|
||||
if err != nil {
|
||||
log.Errorf("redis Get dispatch failed: %+v", err)
|
||||
continue
|
||||
}
|
||||
scheduled++
|
||||
}
|
||||
|
||||
if scheduled == 0 {
|
||||
// Nothing dispatched (e.g. cluster missing) — don't hang the response.
|
||||
return types.ActionContinue
|
||||
}
|
||||
return types.ActionPause
|
||||
}
|
||||
Reference in New Issue
Block a user