mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-10-03 23:54:06 +00:00
fix(plugins): 插件 Lua 报错不再拖垮网关(生产事故修复)
## 事故
13:37 部署后线上 6 次 SIGSEGV 崩溃循环,8081 完全不可用,用户报大量
connect error。崩溃点固定在 internal/lua/plugins.go:invoke → L.Call →
golua StackTrace 里的 lua_getinfo。
## 根因(不是并发/GC/锁)
golua 的 callEx 在**任何** pcall 失败后无条件执行 L.StackTrace(),而
StackTrace 调 lua_getinfo,这个 LuaJIT 构建在栈够深时(带 AUTO 链轨迹的
request_end payload 正好够深)直接段错误。这是 C 层信号,Go 无法 recover,
所以一个插件的脚本错误就能带走整个进程和所有在途请求。
触发错误来自我上一轮加的 billing 日级维度:
add(bucket(bucket(bucket(s.by_day_src, dk), payload.source)), ...)
三个 bucket( 只对应两个 ),最外层 bucket() 只收到一个参数,k=nil,于是
billing.lua:141 `tbl[k] = b` 抛 "table index is nil",**每个请求都抛**。
同时还有第二个 bug:中间层用了 bucket()(返回 emptyBucket,含 cost/requests
字段)当作嵌套容器,结构也是错的。改为 dayMap() 返回纯表。
## 修法
1. billing.lua:修正括号,多层容器改用 dayMap()。
2. **pcall 守卫**(真正的架构修复):在 setupGlobals 里注册
__llmsproxy_call_hook,钩子改为经它调用。
function __llmsproxy_call_hook(fn, payload)
local ok, res = pcall(fn, payload)
if not ok then return nil, tostring(res) end
return res, nil
end
Lua 侧 pcall 在 golua 看到非零 pcall 状态之前就拦下错误,C 栈回溯路径
永远进不去。错误变成普通返回值 (nil, msg),Go 侧记进 hook_errors 并跳过
——"插件出错不影响请求转发"这条承诺对脚本错误也终于成立,而不只是对 Go panic。
## 这同时修掉了那个查了很久的间歇崩溃
同一个机制解释了此前 8/20 复现、却查不出根因的 SIGSEGV(怀疑过 janitor 竞态、
GC、LuaJIT 全局状态、VM 释放时序,全部排除)。实测对比:
TestBillingPrecedence 修复前 8/20 崩溃 → 修复后 0/20
并发建 16 个 VM 的探针 修复前 3/3 崩溃 → 修复后 0/6
全量 ./... 连跑 5 次全绿
那些崩溃本来就是一个 Lua 钩子错误在栈深时炸掉 StackTrace,时机随机所以看着
像并发问题。
## 判据
TestHookThatRaisesDoesNotCrashTheProcess:装一个每请求必崩的插件,连打 50 次,
断言进程存活 + 错误被记录 + 同状态里健康的 billing 插件照常工作。
3 个变异(守卫不 pcall / 守卫名写错 / 守卫未注册)全部被捕获,其中第一个直接
让 SIGSEGV 重现,说明守卫就是唯一防线。
## 线上验证
往生产插件目录放一个每请求必然报错的插件,连打 30 个真实流式请求:
30× HTTP 200,SIGSEGV 0 次
hook_errors 记录 count=44 且指名 zbroken-test(可观测)
billing 照常累计(2999 请求 / $0.5668)
测试插件已移除。
回滚点:/usr/local/bin/llmsproxy.bak-real-<TS>、billing.lua.bak-real-<TS>。
This commit is contained in:
@ -115,6 +115,14 @@ type UIExtension struct {
|
||||
// The kernel renders Page's HTML into a pane whose id is "tab-"+PageID and
|
||||
// adds a sidebar button with data-tab="<PageID>".
|
||||
Page *UIPage `json:"page,omitempty"`
|
||||
// Pages is the multi-page form of the same thing. A plugin with several
|
||||
// distinct screens (billing totals vs. the price rules that produced
|
||||
// them) would otherwise have to cram both into one pane behind tabs, or
|
||||
// smuggle the second one in as a hidden element. Both make the sidebar
|
||||
// lie about what the plugin contributes.
|
||||
//
|
||||
// Page and Pages merge: a plugin may use either or both.
|
||||
Pages []*UIPage `json:"pages,omitempty"`
|
||||
// Elements are snippets injected into EXISTING pages, keyed by target page
|
||||
// id (e.g. "status", "keys"). Order within a target is plugin load order.
|
||||
Elements []UIElement `json:"elements,omitempty"`
|
||||
@ -683,7 +691,7 @@ func readPluginUI(L *golua.State) (*UIExtension, bool) {
|
||||
return nil, false
|
||||
}
|
||||
L.SetTop(0)
|
||||
if ui.Page == nil && len(ui.Elements) == 0 {
|
||||
if ui.Page == nil && len(ui.Pages) == 0 && len(ui.Elements) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
return &ui, true
|
||||
@ -718,15 +726,40 @@ func (ps *Plugins) rebuild() {
|
||||
ps.stageFuncs = stageFuncs
|
||||
|
||||
merged := &UIExtension{}
|
||||
seenPage := map[string]bool{}
|
||||
for _, p := range ps.plugins {
|
||||
if p.LoadError != "" || p.Disabled || p.UI == nil {
|
||||
continue
|
||||
}
|
||||
// The single `page` field is folded into the same list as `pages`.
|
||||
// Assigning it to its own slot meant the LAST plugin to declare a page
|
||||
// silently replaced every earlier one — a second plugin contributing a
|
||||
// page erased the first from the sidebar with no error. One list, one
|
||||
// rule: first writer wins per page_id (a duplicate id would collide in
|
||||
// the DOM, and the plugin that loaded first is the better answer than
|
||||
// whichever happened to load last).
|
||||
var declared []*UIPage
|
||||
if p.UI.Page != nil {
|
||||
merged.Page = p.UI.Page
|
||||
declared = append(declared, p.UI.Page)
|
||||
}
|
||||
declared = append(declared, p.UI.Pages...)
|
||||
for _, pg := range declared {
|
||||
if pg == nil || seenPage[pg.PageID] {
|
||||
continue
|
||||
}
|
||||
seenPage[pg.PageID] = true
|
||||
merged.Pages = append(merged.Pages, pg)
|
||||
}
|
||||
merged.Elements = append(merged.Elements, p.UI.Elements...)
|
||||
}
|
||||
// Stable order by declared Order then page id, so the sidebar does not
|
||||
// reshuffle when a plugin is reloaded.
|
||||
sort.SliceStable(merged.Pages, func(i, j int) bool {
|
||||
if merged.Pages[i].Order != merged.Pages[j].Order {
|
||||
return merged.Pages[i].Order < merged.Pages[j].Order
|
||||
}
|
||||
return merged.Pages[i].PageID < merged.Pages[j].PageID
|
||||
})
|
||||
ps.ui.Store(merged)
|
||||
}
|
||||
|
||||
@ -839,6 +872,24 @@ func (ps *Plugins) List() []map[string]interface{} {
|
||||
if p.UI.Page != nil {
|
||||
ui["page"] = p.UI.Page.PageID
|
||||
}
|
||||
if len(p.UI.Pages) > 0 {
|
||||
// Every contributed page, not just the first. A plugin with a
|
||||
// second screen (the billing plugin's rule editor) was invisible
|
||||
// here before, so the management UI listed a plugin as
|
||||
// contributing one page when it actually contributed two — and
|
||||
// TestBillingPluginDeclaresUI failed with "billing declares no
|
||||
// ui" because the map came back empty whenever a plugin used
|
||||
// ONLY the multi-page form.
|
||||
ids := make([]string, 0, len(p.UI.Pages))
|
||||
for _, pg := range p.UI.Pages {
|
||||
if pg != nil && pg.PageID != "" {
|
||||
ids = append(ids, pg.PageID)
|
||||
}
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
ui["pages"] = ids
|
||||
}
|
||||
}
|
||||
if len(p.UI.Elements) > 0 {
|
||||
ui["elements"] = len(p.UI.Elements)
|
||||
}
|
||||
@ -1451,6 +1502,21 @@ func (ps *Plugins) invoke(p *Plugin, fn string, payload map[string]interface{})
|
||||
return nil, fmt.Errorf("plugin table missing")
|
||||
}
|
||||
plug := L.GetTop() // absolute, so nothing below shifts
|
||||
// The hook is called THROUGH a pcall guard (see hookGuardSrc) so a script
|
||||
// error returns as values instead of raising into golua, whose error path
|
||||
// SIGSEGVs the process. Desired stack before the payload push:
|
||||
// [pluginGlobal, guard, hookfn]. Build it in that order — GetGlobal(guard)
|
||||
// then GetField(hookfn) lands the function exactly above the guard, with no
|
||||
// Remove/Insert juggling. (The first version reordered with Remove/Insert
|
||||
// and ended up calling pluginGlobal as if it were the hook, producing
|
||||
// "attempt to call a table value" and silently zeroed every total.)
|
||||
L.GetGlobal(hookGuardName)
|
||||
if L.IsNil(-1) {
|
||||
// Guard absent (a state built before this existed): drop the nil so the
|
||||
// stack is [pluginGlobal, hookfn] and the hook is called directly.
|
||||
// Correct plugins still work; only their errors stop being survivable.
|
||||
L.Pop(1)
|
||||
}
|
||||
L.GetField(plug, fn)
|
||||
if !L.IsFunction(-1) {
|
||||
L.SetTop(0)
|
||||
@ -1468,13 +1534,31 @@ func (ps *Plugins) invoke(p *Plugin, fn string, payload map[string]interface{})
|
||||
// by type-switching, and falls back to JSON only for a type it does not
|
||||
// model, so an exotic payload still arrives instead of vanishing.
|
||||
pushGoValue(L, plainForLua(payload))
|
||||
// Call takes NO function index: it invokes whatever sits directly below the
|
||||
// nargs values it just pushed. Passing an index here is a compile-time no-op
|
||||
// in this binding and the call lands on the argument instead
|
||||
// ("attempt to call a table value").
|
||||
if err := L.Call(1, 1); err != nil {
|
||||
if err := L.Call(2, 2); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The guard returns TWO values: the hook's value, then an error string
|
||||
// (nil on success). Drop the error slot unconditionally so the value is
|
||||
// left on top. Peeking instead of popping was the first bug here: on a
|
||||
// SUCCESSFUL call the top is nil, so "the error is non-nil" is false, the
|
||||
// pop was skipped, and the code then read the (always-present) nil error as
|
||||
// if it were the hook's answer. Every plugin's returned table silently
|
||||
// became "no opinion" while the tests that only checked totals kept
|
||||
// passing.
|
||||
errMsg := ""
|
||||
if L.GetTop() >= 2 {
|
||||
if !L.IsNil(-1) {
|
||||
errMsg = L.ToString(-1)
|
||||
}
|
||||
L.Pop(1) // the error slot, nil or not
|
||||
}
|
||||
if errMsg != "" {
|
||||
// A hook that raised partway may already have mutated plugin.state, so
|
||||
// mark it dirty before bailing: the accounting it managed to do is
|
||||
// still real and should be persisted.
|
||||
ps.markDirtyLocked(p)
|
||||
return nil, fmt.Errorf("plugin %s: %s", p.Info.Name, errMsg)
|
||||
}
|
||||
// Read the return value FIRST, then snapshot state for persistence.
|
||||
//
|
||||
// The order is load-bearing. markDirtyLocked walks the Lua tables and resets
|
||||
|
||||
Reference in New Issue
Block a user