mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-10-03 23:54:06 +00:00
## 事故
13:37 部署后线上 6 次 SIGSEGV 崩溃循环,8081 完全不可用,用户报大量
connect error。崩溃点固定在 internal/lua/plugins.go:invoke → L.Call →
golua StackTrace 里的 lua_getinfo。
## 根因(不是并发/GC/锁)
golua 的 callEx 在**任何** pcall 失败后无条件执行 L.StackTrace(),而
StackTrace 调 lua_getinfo,这个 LuaJIT 构建在栈够深时(带 AUTO 链轨迹的
request_end payload 正好够深)直接段错误。这是 C 层信号,Go 无法 recover,
所以一个插件的脚本错误就能带走整个进程和所有在途请求。
触发错误来自我上一轮加的 billing 日级维度:
add(bucket(bucket(bucket(s.by_day_src, dk), payload.source)), ...)
三个 bucket( 只对应两个 ),最外层 bucket() 只收到一个参数,k=nil,于是
billing.lua:141 `tbl[k] = b` 抛 "table index is nil",**每个请求都抛**。
同时还有第二个 bug:中间层用了 bucket()(返回 emptyBucket,含 cost/requests
字段)当作嵌套容器,结构也是错的。改为 dayMap() 返回纯表。
## 修法
1. billing.lua:修正括号,多层容器改用 dayMap()。
2. **pcall 守卫**(真正的架构修复):在 setupGlobals 里注册
__llmsproxy_call_hook,钩子改为经它调用。
function __llmsproxy_call_hook(fn, payload)
local ok, res = pcall(fn, payload)
if not ok then return nil, tostring(res) end
return res, nil
end
Lua 侧 pcall 在 golua 看到非零 pcall 状态之前就拦下错误,C 栈回溯路径
永远进不去。错误变成普通返回值 (nil, msg),Go 侧记进 hook_errors 并跳过
——"插件出错不影响请求转发"这条承诺对脚本错误也终于成立,而不只是对 Go panic。
## 这同时修掉了那个查了很久的间歇崩溃
同一个机制解释了此前 8/20 复现、却查不出根因的 SIGSEGV(怀疑过 janitor 竞态、
GC、LuaJIT 全局状态、VM 释放时序,全部排除)。实测对比:
TestBillingPrecedence 修复前 8/20 崩溃 → 修复后 0/20
并发建 16 个 VM 的探针 修复前 3/3 崩溃 → 修复后 0/6
全量 ./... 连跑 5 次全绿
那些崩溃本来就是一个 Lua 钩子错误在栈深时炸掉 StackTrace,时机随机所以看着
像并发问题。
## 判据
TestHookThatRaisesDoesNotCrashTheProcess:装一个每请求必崩的插件,连打 50 次,
断言进程存活 + 错误被记录 + 同状态里健康的 billing 插件照常工作。
3 个变异(守卫不 pcall / 守卫名写错 / 守卫未注册)全部被捕获,其中第一个直接
让 SIGSEGV 重现,说明守卫就是唯一防线。
## 线上验证
往生产插件目录放一个每请求必然报错的插件,连打 30 个真实流式请求:
30× HTTP 200,SIGSEGV 0 次
hook_errors 记录 count=44 且指名 zbroken-test(可观测)
billing 照常累计(2999 请求 / $0.5668)
测试插件已移除。
回滚点:/usr/local/bin/llmsproxy.bak-real-<TS>、billing.lua.bak-real-<TS>。
213 lines
7.3 KiB
Go
213 lines
7.3 KiB
Go
package gateway
|
|
|
|
import (
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"llmsproxy/internal/config"
|
|
"llmsproxy/internal/core"
|
|
"llmsproxy/internal/lua"
|
|
)
|
|
|
|
// gatewayWithBilling boots a gateway with the bundled billing plugin loaded, so
|
|
// the UI-injection endpoint is exercised against a real plugin rather than a
|
|
// hand-written stub. The other plugin tests in this package assert on the
|
|
// WebUI source; this one asserts on the HTTP contract the browser consumes.
|
|
func gatewayWithBilling(t *testing.T) *Gateway {
|
|
t.Helper()
|
|
dir := t.TempDir()
|
|
cfgPath := filepath.Join(dir, "config.yaml")
|
|
body := "listen: :0\n" +
|
|
"adapter_dir: " + filepath.Join(dir, "adapters") + "\n" +
|
|
"plugin_dir: " + filepath.Join(dir, "plugins") + "\n" +
|
|
"runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" +
|
|
"gateway_keys:\n - sk-test\n"
|
|
if err := os.WriteFile(cfgPath, []byte(body), 0600); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cfg, err := config.Load(cfgPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
c, err := core.NewFromConfig(cfg)
|
|
if err != nil {
|
|
t.Fatalf("core: %v", err)
|
|
}
|
|
t.Cleanup(c.Close)
|
|
// Load the shipped plugin explicitly: seeding only runs for a directory that
|
|
// does not exist yet, and this test wants a known plugin regardless.
|
|
src, err := lua.ReadBundledPlugin("billing")
|
|
if err != nil {
|
|
t.Fatalf("read bundled billing: %v", err)
|
|
}
|
|
if err := c.Plugins().LoadSource("billing", src); err != nil {
|
|
t.Fatalf("load billing: %v", err)
|
|
}
|
|
g, err := New(c)
|
|
if err != nil {
|
|
t.Fatalf("gateway: %v", err)
|
|
}
|
|
return g
|
|
}
|
|
|
|
// TestUIInjectServesPluginUI: GET /api/ui-inject is the single call the WebUI
|
|
// makes at boot, and it must carry BOTH a contributed page and contributed
|
|
// elements — the browser builds the sidebar from the page and mounts the
|
|
// elements into existing panes from the same payload, so a partial response
|
|
// would produce a page with no body or a missing tile.
|
|
func TestUIInjectServesPluginUI(t *testing.T) {
|
|
g := gatewayWithBilling(t)
|
|
rr := doReq(t, g, http.MethodGet, "/api/ui-inject", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
|
}
|
|
var view struct {
|
|
// Decoded into the real types so the test cannot drift from the wire
|
|
// contract. An inline copy missed the `pages` field when the payload
|
|
// shape changed and failed to compile, which is at least loud — but the
|
|
// same copy also went on asserting the OLD single-page shape for a
|
|
// release, silently.
|
|
UI lua.UIExtension `json:"ui"`
|
|
Stages []string `json:"stages"`
|
|
}
|
|
if err := json.Unmarshal(rr.Body.Bytes(), &view); err != nil {
|
|
t.Fatalf("decode: %v", err)
|
|
}
|
|
// The payload carries every page in one list; ui.page is no longer a
|
|
// separate slot (it was, and a second plugin contributing a page overwrote
|
|
// whatever was there).
|
|
var billing *lua.UIPage
|
|
for i, pg := range view.UI.Pages {
|
|
if pg != nil && pg.PageID == "billing" {
|
|
billing = view.UI.Pages[i]
|
|
}
|
|
}
|
|
if billing == nil {
|
|
t.Fatalf("no billing page in the inject payload; pages=%d", len(view.UI.Pages))
|
|
}
|
|
if !strings.Contains(billing.Mount, "billing-root") {
|
|
t.Error("the page mount came back empty")
|
|
}
|
|
if len(view.UI.Elements) == 0 {
|
|
t.Error("billing contributes an element to the status page but it is missing")
|
|
}
|
|
for _, e := range view.UI.Elements {
|
|
if e.Target != "status" {
|
|
t.Errorf("element target = %q, want \"status\"", e.Target)
|
|
}
|
|
}
|
|
// Derived from AllStages, not hardcoded: the previous assertion of "3"
|
|
// is exactly what let chain_step go missing from this payload unnoticed.
|
|
if len(view.Stages) != len(lua.AllStages) {
|
|
t.Errorf("stages = %v, want %d (one per AllStages entry)", view.Stages, len(lua.AllStages))
|
|
}
|
|
for i, st := range lua.AllStages {
|
|
if i >= len(view.Stages) || view.Stages[i] != string(st) {
|
|
t.Errorf("stages[%d] = %v, want %q", i, view.Stages, string(st))
|
|
}
|
|
}
|
|
if !containsStr(view.Stages, string(lua.StageChainStep)) {
|
|
t.Error("the discovery payload does not advertise chain_step; a plugin " +
|
|
"author would conclude the stage does not exist")
|
|
}
|
|
}
|
|
|
|
// TestUIInjectIsEmptyWithoutPlugins: a gateway with no plugins must still answer
|
|
// 200 with an empty (not missing, not null) payload. The WebUI calls this
|
|
// unconditionally at boot, so a 404 or a null `ui` would break every dashboard.
|
|
func TestUIInjectIsEmptyWithoutPlugins(t *testing.T) {
|
|
g := newTestGateway(t)
|
|
rr := doReq(t, g, http.MethodGet, "/api/ui-inject", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
|
}
|
|
if !strings.Contains(rr.Body.String(), `"ui"`) {
|
|
t.Error("no ui key in the response; the WebUI would have nothing to read")
|
|
}
|
|
}
|
|
|
|
// containsStr reports whether list has s.
|
|
func containsStr(list []string, s string) bool {
|
|
for _, x := range list {
|
|
if x == s {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// newRecorderFor pushes a request through the full handler chain.
|
|
func newRecorderFor(t *testing.T, g *Gateway, req *http.Request) *httptest.ResponseRecorder {
|
|
t.Helper()
|
|
rr := httptest.NewRecorder()
|
|
g.Handler().ServeHTTP(rr, req)
|
|
return rr
|
|
}
|
|
|
|
// TestPluginsListAndStateAPI covers the management surface the plugin docs
|
|
// promise: listing, and reading a plugin's own published state.
|
|
func TestPluginsListAndStateAPI(t *testing.T) {
|
|
g := gatewayWithBilling(t)
|
|
rr := doReq(t, g, http.MethodGet, "/api/plugins", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("GET /api/plugins = %d: %s", rr.Code, rr.Body.String())
|
|
}
|
|
for _, want := range []string{"billing", "hook_errors", "plugin_dir", "request_end"} {
|
|
if !strings.Contains(rr.Body.String(), want) {
|
|
t.Errorf("/api/plugins response lacks %q", want)
|
|
}
|
|
}
|
|
// state read: the plugin published its (empty) state, so the key exists.
|
|
rr = doReq(t, g, http.MethodGet, "/api/plugins/billing/state", "")
|
|
if rr.Code != http.StatusOK {
|
|
t.Fatalf("GET state = %d: %s", rr.Code, rr.Body.String())
|
|
}
|
|
if !strings.Contains(rr.Body.String(), `"total"`) {
|
|
t.Errorf("billing state lacks the total bucket: %s", rr.Body.String())
|
|
}
|
|
}
|
|
|
|
// TestPluginStatePUTIsAdminOnly: only the WRITE side is gated. A user key must
|
|
// be able to READ its own billing widget's data, but must not be able to
|
|
// rewrite the price table.
|
|
func TestPluginStatePUTIsAdminOnly(t *testing.T) {
|
|
g := gatewayWithBilling(t)
|
|
|
|
// Build a user-role key and remember its secret.
|
|
rec, err := g.core.CreateKey("viewer", "user", nil, "")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
userKey := rec.Key
|
|
|
|
// A user key may read the state.
|
|
req, _ := http.NewRequest(http.MethodGet, "/api/plugins/billing/state", nil)
|
|
req.Header.Set("Authorization", "Bearer "+userKey)
|
|
rr := newRecorderFor(t, g, req)
|
|
if rr.Code != http.StatusOK {
|
|
t.Errorf("user GET state = %d, want 200 (the billing widget must render for users)", rr.Code)
|
|
}
|
|
|
|
// A user key may NOT write it.
|
|
put, _ := http.NewRequest(http.MethodPut, "/api/plugins/billing/state",
|
|
strings.NewReader(`{"prices":{"default":{"prompt":0}}}`))
|
|
put.Header.Set("Authorization", "Bearer "+userKey)
|
|
put.Header.Set("Content-Type", "application/json")
|
|
prr := newRecorderFor(t, g, put)
|
|
if prr.Code != http.StatusForbidden {
|
|
t.Errorf("user PUT state = %d, want 403 (a user must not rewrite the price table)", prr.Code)
|
|
}
|
|
|
|
// An admin key may.
|
|
adm := doReq(t, g, http.MethodPut, "/api/plugins/billing/state",
|
|
`{"prices":{"default":{"prompt":1e-6}}}`)
|
|
if adm.Code != http.StatusOK {
|
|
t.Errorf("admin PUT state = %d: %s", adm.Code, adm.Body.String())
|
|
}
|
|
}
|