diff --git a/plugins/wasm-go/extensions/test-redis-inject-spin/VERSION b/plugins/wasm-go/extensions/test-redis-inject-spin/VERSION new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/plugins/wasm-go/extensions/test-redis-inject-spin/VERSION @@ -0,0 +1 @@ +1 diff --git a/plugins/wasm-go/extensions/test-redis-inject-spin/go.mod b/plugins/wasm-go/extensions/test-redis-inject-spin/go.mod new file mode 100644 index 000000000..434c63763 --- /dev/null +++ b/plugins/wasm-go/extensions/test-redis-inject-spin/go.mod @@ -0,0 +1,20 @@ +module github.com/alibaba/higress/plugins/wasm-go/extensions/test-redis-inject-spin + +go 1.24.1 + +toolchain go1.24.4 + +require ( + github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2 + github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d + github.com/tidwall/gjson v1.18.0 + github.com/tidwall/resp v0.1.1 + google.golang.org/protobuf v1.36.6 +) + +require ( + github.com/google/uuid v1.6.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/sjson v1.2.5 // indirect +) diff --git a/plugins/wasm-go/extensions/test-redis-inject-spin/go.sum b/plugins/wasm-go/extensions/test-redis-inject-spin/go.sum new file mode 100644 index 000000000..85992c358 --- /dev/null +++ b/plugins/wasm-go/extensions/test-redis-inject-spin/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2 h1:NY33OrWCJJ+DFiLc+lsBY4Ywor2Ik61ssk6qkGF8Ypo= +github.com/higress-group/proxy-wasm-go-sdk v0.0.0-20251103120604-77e9cce339d2/go.mod h1:tRI2LfMudSkKHhyv1uex3BWzcice2s/l8Ah8axporfA= +github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d h1:LgYbzEBtg0+LEqoebQeMVgAB6H5SgqG+KN+gBhNfKbM= +github.com/higress-group/wasm-go v1.0.10-0.20260120033417-1c84f010156d/go.mod h1:uKVYICbRaxTlKqdm8E0dpjbysxM8uCPb9LV26hF3Km8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/resp v0.1.1 h1:Ly20wkhqKTmDUPlyM1S7pWo5kk0tDu8OoC/vFArXmwE= +github.com/tidwall/resp v0.1.1/go.mod h1:3/FrruOBAxPTPtundW0VXgmsQ4ZBA0Aw714lVYgwFa0= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/plugins/wasm-go/extensions/test-redis-inject-spin/main.go b/plugins/wasm-go/extensions/test-redis-inject-spin/main.go new file mode 100644 index 000000000..f576e3ed1 --- /dev/null +++ b/plugins/wasm-go/extensions/test-redis-inject-spin/main.go @@ -0,0 +1,119 @@ +// Repro plugin for higress issue #4034: worker CPU spin in +// WasmBase::doAfterVmCallActions when a deferred async callback synchronously +// calls injectEncodedDataToFilterChain (nesting SaveRestoreContext) so that the +// after-vm-call action re-queues itself forever. +// +// Repro shape (dead-Redis only, self-contained — no external HTTP dependency): +// 1. In the response-header phase, pause the response. +// 2. Fire N Redis commands against an UNREACHABLE redis cluster. +// 3. Each failure callback (onRedisCallFailure) synchronously calls the +// inject_encoded_data_to_filter_chain foreign function. Because sibling +// failure callbacks are still queued as after-vm-call actions, the nested +// SaveRestoreContext leaves current_context_ != nullptr and the deferred +// action re-queues itself -> CPU spin on the pre-fix host. +// +// With the Layer A drain-to-local fix in place, the drain terminates over a +// snapshot and worker CPU stays bounded. +package main + +import ( + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm" + "github.com/higress-group/proxy-wasm-go-sdk/proxywasm/types" + "github.com/higress-group/wasm-go/pkg/log" + pb "github.com/higress-group/wasm-go/pkg/protos" + "github.com/higress-group/wasm-go/pkg/wrapper" + "github.com/tidwall/gjson" + "github.com/tidwall/resp" + "google.golang.org/protobuf/proto" +) + +func main() {} + +type Config struct { + redisClient *wrapper.RedisClusterClient[wrapper.FQDNCluster] + redisKey string + injectBody string + injectCount int +} + +func init() { + wrapper.SetCtx( + "test-redis-inject-spin", + wrapper.ParseConfig(parseConfig), + wrapper.ProcessResponseHeaders(onHttpResponseHeaders), + ) +} + +func parseConfig(json gjson.Result, config *Config) error { + serviceName := json.Get("service_name").String() + if serviceName == "" { + serviceName = "dead-redis.dead-redis.svc.cluster.local" + } + servicePort := json.Get("service_port").Int() + if servicePort == 0 { + servicePort = 6379 + } + config.redisKey = json.Get("redis_key").String() + if config.redisKey == "" { + config.redisKey = "higress-4034-key" + } + config.injectBody = json.Get("inject_body").String() + if config.injectBody == "" { + config.injectBody = "injected-by-4034-repro\n" + } + config.injectCount = int(json.Get("inject_count").Int()) + if config.injectCount == 0 { + config.injectCount = 16 + } + + config.redisClient = wrapper.NewRedisClusterClient(wrapper.FQDNCluster{ + FQDN: serviceName, + Port: servicePort, + }) + // Init never returns error for an unreachable host; commands' callbacks will + // fire with an error value, which is exactly the #4034 trigger path. + return config.redisClient.Init("", "", 1000) +} + +func onHttpResponseHeaders(ctx wrapper.HttpContext, config Config) types.Action { + proxywasm.RemoveHttpResponseHeader("content-length") + ctx.DontReadResponseBody() + + inject := func() { + d := &pb.InjectEncodedDataToFilterChainArguments{ + Body: config.injectBody, + Endstream: true, + } + s, err := proto.Marshal(d) + if err != nil { + log.Errorf("marshal inject args failed: %+v", err) + return + } + if _, err := proxywasm.CallForeignFunction("inject_encoded_data_to_filter_chain_on_header", s); err != nil { + log.Errorf("call inject_encoded_data_to_filter_chain_on_header failed: %+v", err) + } + } + + scheduled := 0 + for i := 0; i < config.injectCount; i++ { + err := config.redisClient.Get(config.redisKey, func(response resp.Value) { + // Fires on redis-unreachable failure; synchronously inject to nest + // SaveRestoreContext while sibling callbacks are still queued. + if response.Error() != nil { + log.Debugf("redis get failed as expected: %v", response.Error()) + } + inject() + }) + if err != nil { + log.Errorf("redis Get dispatch failed: %+v", err) + continue + } + scheduled++ + } + + if scheduled == 0 { + // Nothing dispatched (e.g. cluster missing) — don't hang the response. + return types.ActionContinue + } + return types.ActionPause +} diff --git a/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.go b/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.go new file mode 100644 index 000000000..fe716ba85 --- /dev/null +++ b/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.go @@ -0,0 +1,75 @@ +// Copyright (c) 2026 Alibaba Group Holding Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tests + +import ( + "testing" + + "github.com/alibaba/higress/v2/test/e2e/conformance/utils/http" + "github.com/alibaba/higress/v2/test/e2e/conformance/utils/suite" +) + +func init() { + Register(WasmPluginsRedisInjectSpin) +} + +// WasmPluginsRedisInjectSpin reproduces issue #4034: a deferred async Redis +// failure callback synchronously calls injectEncodedDataToFilterChain, which +// (pre-fix) makes WasmBase::doAfterVmCallActions re-queue an action forever and +// spins a worker at 100% CPU. Redis is pointed at an unroutable endpoint so the +// failure callbacks fire under concurrency. +// +// With the Layer A drain-to-local fix in place, the after-vm-call drain +// terminates over a snapshot, deferred callbacks complete, and the gateway keeps +// serving requests. The test asserts the plugin route stays responsive (200) +// rather than hanging — a spinning worker would fail this via timeout. +var WasmPluginsRedisInjectSpin = suite.ConformanceTest{ + ShortName: "WasmPluginsRedisInjectSpin", + Description: "Reproduce #4034: redis-failure callback + inject must not spin the worker; gateway stays responsive.", + Manifests: []string{"tests/go-wasm-test-redis-inject-spin.yaml"}, + Features: []suite.SupportedFeature{suite.WASMGoConformanceFeature}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + testcases := []http.Assertion{ + { + Meta: http.AssertionMeta{ + TargetBackend: "infra-backend-v1", + TargetNamespace: "higress-conformance-infra", + }, + Request: http.AssertionRequest{ + ActualRequest: http.Request{ + Host: "redis-inject-spin.com", + Path: "/", + UnfollowRedirect: true, + }, + }, + Response: http.AssertionResponse{ + ExpectedResponse: http.Response{ + StatusCode: 200, + }, + }, + }, + } + t.Run("WasmPlugins redis-inject-spin (#4034 CPU spin repro)", func(t *testing.T) { + // Drive repeated requests: on the pre-fix host the deferred callback + // re-queues forever and a worker spins, so the route stops + // responding; on the fixed host every request completes. + for i := 0; i < 20; i++ { + for _, testcase := range testcases { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, suite.GatewayAddress, testcase) + } + } + }) + }, +} diff --git a/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.yaml b/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.yaml new file mode 100644 index 000000000..e5da69bd6 --- /dev/null +++ b/test/e2e/conformance/tests/go-wasm-test-redis-inject-spin.yaml @@ -0,0 +1,75 @@ +# Copyright (c) 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A "dead" Redis: the Service (with manual Endpoints) makes Higress create the +# envoy cluster so RedisInit succeeds and commands dispatch, but the endpoint IP +# is unroutable (240.0.0.1, class-E) so every connection fails -> the plugin's +# redis-failure callbacks fire, which is the #4034 trigger. +apiVersion: v1 +kind: Service +metadata: + name: dead-redis + namespace: higress-conformance-infra +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: 6379 +--- +apiVersion: v1 +kind: Endpoints +metadata: + name: dead-redis + namespace: higress-conformance-infra +subsets: + - addresses: + - ip: 240.0.0.1 + ports: + - name: redis + port: 6379 + protocol: TCP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: redis-inject-spin + namespace: higress-conformance-infra +spec: + ingressClassName: higress + rules: + - host: "redis-inject-spin.com" + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: infra-backend-v1 + port: + number: 8080 +--- +apiVersion: extensions.higress.io/v1alpha1 +kind: WasmPlugin +metadata: + name: test-redis-inject-spin + namespace: higress-system +spec: + defaultConfig: + service_name: "dead-redis.higress-conformance-infra.svc.cluster.local" + service_port: 6379 + redis_key: "higress-4034-key" + inject_body: "injected-by-4034-repro\n" + inject_count: 16 + url: file:///opt/plugins/wasm-go/extensions/test-redis-inject-spin/plugin.wasm