Files
ModelRouter/internal/gateway/plugins_api_test.go
JianFeeeee d9652f479a fix(plugins): 插件 Lua 报错不再拖垮网关(生产事故修复)
## 事故

13:37 部署后线上 6 次 SIGSEGV 崩溃循环,8081 完全不可用,用户报大量
connect error。崩溃点固定在 internal/lua/plugins.go:invoke → L.Call →
golua StackTrace 里的 lua_getinfo。

## 根因(不是并发/GC/锁)

golua 的 callEx 在**任何** pcall 失败后无条件执行 L.StackTrace(),而
StackTrace 调 lua_getinfo,这个 LuaJIT 构建在栈够深时(带 AUTO 链轨迹的
request_end payload 正好够深)直接段错误。这是 C 层信号,Go 无法 recover,
所以一个插件的脚本错误就能带走整个进程和所有在途请求。

触发错误来自我上一轮加的 billing 日级维度:

    add(bucket(bucket(bucket(s.by_day_src, dk), payload.source)), ...)

三个 bucket( 只对应两个 ),最外层 bucket() 只收到一个参数,k=nil,于是
billing.lua:141 `tbl[k] = b` 抛 "table index is nil",**每个请求都抛**。

同时还有第二个 bug:中间层用了 bucket()(返回 emptyBucket,含 cost/requests
字段)当作嵌套容器,结构也是错的。改为 dayMap() 返回纯表。

## 修法

1. billing.lua:修正括号,多层容器改用 dayMap()。
2. **pcall 守卫**(真正的架构修复):在 setupGlobals 里注册
   __llmsproxy_call_hook,钩子改为经它调用。

       function __llmsproxy_call_hook(fn, payload)
         local ok, res = pcall(fn, payload)
         if not ok then return nil, tostring(res) end
         return res, nil
       end

   Lua 侧 pcall 在 golua 看到非零 pcall 状态之前就拦下错误,C 栈回溯路径
   永远进不去。错误变成普通返回值 (nil, msg),Go 侧记进 hook_errors 并跳过
   ——"插件出错不影响请求转发"这条承诺对脚本错误也终于成立,而不只是对 Go panic。

## 这同时修掉了那个查了很久的间歇崩溃

同一个机制解释了此前 8/20 复现、却查不出根因的 SIGSEGV(怀疑过 janitor 竞态、
GC、LuaJIT 全局状态、VM 释放时序,全部排除)。实测对比:

  TestBillingPrecedence   修复前 8/20 崩溃 → 修复后 0/20
  并发建 16 个 VM 的探针   修复前 3/3  崩溃 → 修复后 0/6
  全量 ./...              连跑 5 次全绿

那些崩溃本来就是一个 Lua 钩子错误在栈深时炸掉 StackTrace,时机随机所以看着
像并发问题。

## 判据

TestHookThatRaisesDoesNotCrashTheProcess:装一个每请求必崩的插件,连打 50 次,
断言进程存活 + 错误被记录 + 同状态里健康的 billing 插件照常工作。
3 个变异(守卫不 pcall / 守卫名写错 / 守卫未注册)全部被捕获,其中第一个直接
让 SIGSEGV 重现,说明守卫就是唯一防线。

## 线上验证

往生产插件目录放一个每请求必然报错的插件,连打 30 个真实流式请求:

  30× HTTP 200,SIGSEGV 0 次
  hook_errors 记录 count=44 且指名 zbroken-test(可观测)
  billing 照常累计(2999 请求 / $0.5668)

测试插件已移除。

回滚点:/usr/local/bin/llmsproxy.bak-real-<TS>、billing.lua.bak-real-<TS>。
2026-10-02 14:07:44 +08:00

213 lines
7.3 KiB
Go

package gateway
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"llmsproxy/internal/config"
"llmsproxy/internal/core"
"llmsproxy/internal/lua"
)
// gatewayWithBilling boots a gateway with the bundled billing plugin loaded, so
// the UI-injection endpoint is exercised against a real plugin rather than a
// hand-written stub. The other plugin tests in this package assert on the
// WebUI source; this one asserts on the HTTP contract the browser consumes.
func gatewayWithBilling(t *testing.T) *Gateway {
t.Helper()
dir := t.TempDir()
cfgPath := filepath.Join(dir, "config.yaml")
body := "listen: :0\n" +
"adapter_dir: " + filepath.Join(dir, "adapters") + "\n" +
"plugin_dir: " + filepath.Join(dir, "plugins") + "\n" +
"runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" +
"gateway_keys:\n - sk-test\n"
if err := os.WriteFile(cfgPath, []byte(body), 0600); err != nil {
t.Fatal(err)
}
cfg, err := config.Load(cfgPath)
if err != nil {
t.Fatal(err)
}
c, err := core.NewFromConfig(cfg)
if err != nil {
t.Fatalf("core: %v", err)
}
t.Cleanup(c.Close)
// Load the shipped plugin explicitly: seeding only runs for a directory that
// does not exist yet, and this test wants a known plugin regardless.
src, err := lua.ReadBundledPlugin("billing")
if err != nil {
t.Fatalf("read bundled billing: %v", err)
}
if err := c.Plugins().LoadSource("billing", src); err != nil {
t.Fatalf("load billing: %v", err)
}
g, err := New(c)
if err != nil {
t.Fatalf("gateway: %v", err)
}
return g
}
// TestUIInjectServesPluginUI: GET /api/ui-inject is the single call the WebUI
// makes at boot, and it must carry BOTH a contributed page and contributed
// elements — the browser builds the sidebar from the page and mounts the
// elements into existing panes from the same payload, so a partial response
// would produce a page with no body or a missing tile.
func TestUIInjectServesPluginUI(t *testing.T) {
g := gatewayWithBilling(t)
rr := doReq(t, g, http.MethodGet, "/api/ui-inject", "")
if rr.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
}
var view struct {
// Decoded into the real types so the test cannot drift from the wire
// contract. An inline copy missed the `pages` field when the payload
// shape changed and failed to compile, which is at least loud — but the
// same copy also went on asserting the OLD single-page shape for a
// release, silently.
UI lua.UIExtension `json:"ui"`
Stages []string `json:"stages"`
}
if err := json.Unmarshal(rr.Body.Bytes(), &view); err != nil {
t.Fatalf("decode: %v", err)
}
// The payload carries every page in one list; ui.page is no longer a
// separate slot (it was, and a second plugin contributing a page overwrote
// whatever was there).
var billing *lua.UIPage
for i, pg := range view.UI.Pages {
if pg != nil && pg.PageID == "billing" {
billing = view.UI.Pages[i]
}
}
if billing == nil {
t.Fatalf("no billing page in the inject payload; pages=%d", len(view.UI.Pages))
}
if !strings.Contains(billing.Mount, "billing-root") {
t.Error("the page mount came back empty")
}
if len(view.UI.Elements) == 0 {
t.Error("billing contributes an element to the status page but it is missing")
}
for _, e := range view.UI.Elements {
if e.Target != "status" {
t.Errorf("element target = %q, want \"status\"", e.Target)
}
}
// Derived from AllStages, not hardcoded: the previous assertion of "3"
// is exactly what let chain_step go missing from this payload unnoticed.
if len(view.Stages) != len(lua.AllStages) {
t.Errorf("stages = %v, want %d (one per AllStages entry)", view.Stages, len(lua.AllStages))
}
for i, st := range lua.AllStages {
if i >= len(view.Stages) || view.Stages[i] != string(st) {
t.Errorf("stages[%d] = %v, want %q", i, view.Stages, string(st))
}
}
if !containsStr(view.Stages, string(lua.StageChainStep)) {
t.Error("the discovery payload does not advertise chain_step; a plugin " +
"author would conclude the stage does not exist")
}
}
// TestUIInjectIsEmptyWithoutPlugins: a gateway with no plugins must still answer
// 200 with an empty (not missing, not null) payload. The WebUI calls this
// unconditionally at boot, so a 404 or a null `ui` would break every dashboard.
func TestUIInjectIsEmptyWithoutPlugins(t *testing.T) {
g := newTestGateway(t)
rr := doReq(t, g, http.MethodGet, "/api/ui-inject", "")
if rr.Code != http.StatusOK {
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
}
if !strings.Contains(rr.Body.String(), `"ui"`) {
t.Error("no ui key in the response; the WebUI would have nothing to read")
}
}
// containsStr reports whether list has s.
func containsStr(list []string, s string) bool {
for _, x := range list {
if x == s {
return true
}
}
return false
}
// newRecorderFor pushes a request through the full handler chain.
func newRecorderFor(t *testing.T, g *Gateway, req *http.Request) *httptest.ResponseRecorder {
t.Helper()
rr := httptest.NewRecorder()
g.Handler().ServeHTTP(rr, req)
return rr
}
// TestPluginsListAndStateAPI covers the management surface the plugin docs
// promise: listing, and reading a plugin's own published state.
func TestPluginsListAndStateAPI(t *testing.T) {
g := gatewayWithBilling(t)
rr := doReq(t, g, http.MethodGet, "/api/plugins", "")
if rr.Code != http.StatusOK {
t.Fatalf("GET /api/plugins = %d: %s", rr.Code, rr.Body.String())
}
for _, want := range []string{"billing", "hook_errors", "plugin_dir", "request_end"} {
if !strings.Contains(rr.Body.String(), want) {
t.Errorf("/api/plugins response lacks %q", want)
}
}
// state read: the plugin published its (empty) state, so the key exists.
rr = doReq(t, g, http.MethodGet, "/api/plugins/billing/state", "")
if rr.Code != http.StatusOK {
t.Fatalf("GET state = %d: %s", rr.Code, rr.Body.String())
}
if !strings.Contains(rr.Body.String(), `"total"`) {
t.Errorf("billing state lacks the total bucket: %s", rr.Body.String())
}
}
// TestPluginStatePUTIsAdminOnly: only the WRITE side is gated. A user key must
// be able to READ its own billing widget's data, but must not be able to
// rewrite the price table.
func TestPluginStatePUTIsAdminOnly(t *testing.T) {
g := gatewayWithBilling(t)
// Build a user-role key and remember its secret.
rec, err := g.core.CreateKey("viewer", "user", nil, "")
if err != nil {
t.Fatal(err)
}
userKey := rec.Key
// A user key may read the state.
req, _ := http.NewRequest(http.MethodGet, "/api/plugins/billing/state", nil)
req.Header.Set("Authorization", "Bearer "+userKey)
rr := newRecorderFor(t, g, req)
if rr.Code != http.StatusOK {
t.Errorf("user GET state = %d, want 200 (the billing widget must render for users)", rr.Code)
}
// A user key may NOT write it.
put, _ := http.NewRequest(http.MethodPut, "/api/plugins/billing/state",
strings.NewReader(`{"prices":{"default":{"prompt":0}}}`))
put.Header.Set("Authorization", "Bearer "+userKey)
put.Header.Set("Content-Type", "application/json")
prr := newRecorderFor(t, g, put)
if prr.Code != http.StatusForbidden {
t.Errorf("user PUT state = %d, want 403 (a user must not rewrite the price table)", prr.Code)
}
// An admin key may.
adm := doReq(t, g, http.MethodPut, "/api/plugins/billing/state",
`{"prices":{"default":{"prompt":1e-6}}}`)
if adm.Code != http.StatusOK {
t.Errorf("admin PUT state = %d: %s", adm.Code, adm.Body.String())
}
}