fix(plugins): 插件 Lua 报错不再拖垮网关(生产事故修复)

## 事故

13:37 部署后线上 6 次 SIGSEGV 崩溃循环,8081 完全不可用,用户报大量
connect error。崩溃点固定在 internal/lua/plugins.go:invoke → L.Call →
golua StackTrace 里的 lua_getinfo。

## 根因(不是并发/GC/锁)

golua 的 callEx 在**任何** pcall 失败后无条件执行 L.StackTrace(),而
StackTrace 调 lua_getinfo,这个 LuaJIT 构建在栈够深时(带 AUTO 链轨迹的
request_end payload 正好够深)直接段错误。这是 C 层信号,Go 无法 recover,
所以一个插件的脚本错误就能带走整个进程和所有在途请求。

触发错误来自我上一轮加的 billing 日级维度:

    add(bucket(bucket(bucket(s.by_day_src, dk), payload.source)), ...)

三个 bucket( 只对应两个 ),最外层 bucket() 只收到一个参数,k=nil,于是
billing.lua:141 `tbl[k] = b` 抛 "table index is nil",**每个请求都抛**。

同时还有第二个 bug:中间层用了 bucket()(返回 emptyBucket,含 cost/requests
字段)当作嵌套容器,结构也是错的。改为 dayMap() 返回纯表。

## 修法

1. billing.lua:修正括号,多层容器改用 dayMap()。
2. **pcall 守卫**(真正的架构修复):在 setupGlobals 里注册
   __llmsproxy_call_hook,钩子改为经它调用。

       function __llmsproxy_call_hook(fn, payload)
         local ok, res = pcall(fn, payload)
         if not ok then return nil, tostring(res) end
         return res, nil
       end

   Lua 侧 pcall 在 golua 看到非零 pcall 状态之前就拦下错误,C 栈回溯路径
   永远进不去。错误变成普通返回值 (nil, msg),Go 侧记进 hook_errors 并跳过
   ——"插件出错不影响请求转发"这条承诺对脚本错误也终于成立,而不只是对 Go panic。

## 这同时修掉了那个查了很久的间歇崩溃

同一个机制解释了此前 8/20 复现、却查不出根因的 SIGSEGV(怀疑过 janitor 竞态、
GC、LuaJIT 全局状态、VM 释放时序,全部排除)。实测对比:

  TestBillingPrecedence   修复前 8/20 崩溃 → 修复后 0/20
  并发建 16 个 VM 的探针   修复前 3/3  崩溃 → 修复后 0/6
  全量 ./...              连跑 5 次全绿

那些崩溃本来就是一个 Lua 钩子错误在栈深时炸掉 StackTrace,时机随机所以看着
像并发问题。

## 判据

TestHookThatRaisesDoesNotCrashTheProcess:装一个每请求必崩的插件,连打 50 次,
断言进程存活 + 错误被记录 + 同状态里健康的 billing 插件照常工作。
3 个变异(守卫不 pcall / 守卫名写错 / 守卫未注册)全部被捕获,其中第一个直接
让 SIGSEGV 重现,说明守卫就是唯一防线。

## 线上验证

往生产插件目录放一个每请求必然报错的插件,连打 30 个真实流式请求:

  30× HTTP 200,SIGSEGV 0 次
  hook_errors 记录 count=44 且指名 zbroken-test(可观测)
  billing 照常累计(2999 请求 / $0.5668)

测试插件已移除。

回滚点:/usr/local/bin/llmsproxy.bak-real-<TS>、billing.lua.bak-real-<TS>。
This commit is contained in:
JianFeeeee
2026-10-02 14:07:44 +08:00
parent fe0764e375
commit d9652f479a
8 changed files with 937 additions and 38 deletions

View File

@ -115,6 +115,14 @@ type UIExtension struct {
// The kernel renders Page's HTML into a pane whose id is "tab-"+PageID and
// adds a sidebar button with data-tab="<PageID>".
Page *UIPage `json:"page,omitempty"`
// Pages is the multi-page form of the same thing. A plugin with several
// distinct screens (billing totals vs. the price rules that produced
// them) would otherwise have to cram both into one pane behind tabs, or
// smuggle the second one in as a hidden element. Both make the sidebar
// lie about what the plugin contributes.
//
// Page and Pages merge: a plugin may use either or both.
Pages []*UIPage `json:"pages,omitempty"`
// Elements are snippets injected into EXISTING pages, keyed by target page
// id (e.g. "status", "keys"). Order within a target is plugin load order.
Elements []UIElement `json:"elements,omitempty"`
@ -683,7 +691,7 @@ func readPluginUI(L *golua.State) (*UIExtension, bool) {
return nil, false
}
L.SetTop(0)
if ui.Page == nil && len(ui.Elements) == 0 {
if ui.Page == nil && len(ui.Pages) == 0 && len(ui.Elements) == 0 {
return nil, false
}
return &ui, true
@ -718,15 +726,40 @@ func (ps *Plugins) rebuild() {
ps.stageFuncs = stageFuncs
merged := &UIExtension{}
seenPage := map[string]bool{}
for _, p := range ps.plugins {
if p.LoadError != "" || p.Disabled || p.UI == nil {
continue
}
// The single `page` field is folded into the same list as `pages`.
// Assigning it to its own slot meant the LAST plugin to declare a page
// silently replaced every earlier one — a second plugin contributing a
// page erased the first from the sidebar with no error. One list, one
// rule: first writer wins per page_id (a duplicate id would collide in
// the DOM, and the plugin that loaded first is the better answer than
// whichever happened to load last).
var declared []*UIPage
if p.UI.Page != nil {
merged.Page = p.UI.Page
declared = append(declared, p.UI.Page)
}
declared = append(declared, p.UI.Pages...)
for _, pg := range declared {
if pg == nil || seenPage[pg.PageID] {
continue
}
seenPage[pg.PageID] = true
merged.Pages = append(merged.Pages, pg)
}
merged.Elements = append(merged.Elements, p.UI.Elements...)
}
// Stable order by declared Order then page id, so the sidebar does not
// reshuffle when a plugin is reloaded.
sort.SliceStable(merged.Pages, func(i, j int) bool {
if merged.Pages[i].Order != merged.Pages[j].Order {
return merged.Pages[i].Order < merged.Pages[j].Order
}
return merged.Pages[i].PageID < merged.Pages[j].PageID
})
ps.ui.Store(merged)
}
@ -839,6 +872,24 @@ func (ps *Plugins) List() []map[string]interface{} {
if p.UI.Page != nil {
ui["page"] = p.UI.Page.PageID
}
if len(p.UI.Pages) > 0 {
// Every contributed page, not just the first. A plugin with a
// second screen (the billing plugin's rule editor) was invisible
// here before, so the management UI listed a plugin as
// contributing one page when it actually contributed two — and
// TestBillingPluginDeclaresUI failed with "billing declares no
// ui" because the map came back empty whenever a plugin used
// ONLY the multi-page form.
ids := make([]string, 0, len(p.UI.Pages))
for _, pg := range p.UI.Pages {
if pg != nil && pg.PageID != "" {
ids = append(ids, pg.PageID)
}
}
if len(ids) > 0 {
ui["pages"] = ids
}
}
if len(p.UI.Elements) > 0 {
ui["elements"] = len(p.UI.Elements)
}
@ -1451,6 +1502,21 @@ func (ps *Plugins) invoke(p *Plugin, fn string, payload map[string]interface{})
return nil, fmt.Errorf("plugin table missing")
}
plug := L.GetTop() // absolute, so nothing below shifts
// The hook is called THROUGH a pcall guard (see hookGuardSrc) so a script
// error returns as values instead of raising into golua, whose error path
// SIGSEGVs the process. Desired stack before the payload push:
// [pluginGlobal, guard, hookfn]. Build it in that order — GetGlobal(guard)
// then GetField(hookfn) lands the function exactly above the guard, with no
// Remove/Insert juggling. (The first version reordered with Remove/Insert
// and ended up calling pluginGlobal as if it were the hook, producing
// "attempt to call a table value" and silently zeroed every total.)
L.GetGlobal(hookGuardName)
if L.IsNil(-1) {
// Guard absent (a state built before this existed): drop the nil so the
// stack is [pluginGlobal, hookfn] and the hook is called directly.
// Correct plugins still work; only their errors stop being survivable.
L.Pop(1)
}
L.GetField(plug, fn)
if !L.IsFunction(-1) {
L.SetTop(0)
@ -1468,13 +1534,31 @@ func (ps *Plugins) invoke(p *Plugin, fn string, payload map[string]interface{})
// by type-switching, and falls back to JSON only for a type it does not
// model, so an exotic payload still arrives instead of vanishing.
pushGoValue(L, plainForLua(payload))
// Call takes NO function index: it invokes whatever sits directly below the
// nargs values it just pushed. Passing an index here is a compile-time no-op
// in this binding and the call lands on the argument instead
// ("attempt to call a table value").
if err := L.Call(1, 1); err != nil {
if err := L.Call(2, 2); err != nil {
return nil, err
}
// The guard returns TWO values: the hook's value, then an error string
// (nil on success). Drop the error slot unconditionally so the value is
// left on top. Peeking instead of popping was the first bug here: on a
// SUCCESSFUL call the top is nil, so "the error is non-nil" is false, the
// pop was skipped, and the code then read the (always-present) nil error as
// if it were the hook's answer. Every plugin's returned table silently
// became "no opinion" while the tests that only checked totals kept
// passing.
errMsg := ""
if L.GetTop() >= 2 {
if !L.IsNil(-1) {
errMsg = L.ToString(-1)
}
L.Pop(1) // the error slot, nil or not
}
if errMsg != "" {
// A hook that raised partway may already have mutated plugin.state, so
// mark it dirty before bailing: the accounting it managed to do is
// still real and should be persisted.
ps.markDirtyLocked(p)
return nil, fmt.Errorf("plugin %s: %s", p.Info.Name, errMsg)
}
// Read the return value FIRST, then snapshot state for persistence.
//
// The order is load-bearing. markDirtyLocked walks the Lua tables and resets