jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-10-05 07:02:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(plugin): AUTO 调度轨迹可见（chain_step stage）

Browse Source 
被问"还有 auto 调度相关 stage 呢？"问出来的真实缺口。

## 问题
chainDrive 只返回 (resp, src, model, err)，调用方只知道**最终哪个槽位赢了**。
遍历过程中算出来又丢掉的东西——哪些档被跳过、为什么跳过、哪些槽位硬失败、
哪档全忙——一律不可见。ChainErr 里其实有这些，但**只在全部失败时**才填，
而它是 error 返回值不是记录。于是：

    "tier 1 冷却所以降级到 tier 3"  ==  "tier 1 正常接单"

对插件而言 tier 只是个常量 -2（"resolved by the chain"），信息量为零。而这
恰恰是优先级链存在的全部理由，也是"我那个贵模型为什么没被用"的答案。

## 做法（scheduler 侧零新依赖）
新增 TraceEvent / TraceSink，chainDrive 多一个可选 sink 参数：

  - TraceEvent 是本包的普通 struct，sink 是 func 参数 ⇒ **不新增 import**，
    scheduler 仍然可独立测试
  - sink 为 nil 时每次 emit 只多一次 nil 判断；没有插件的网关在 AUTO 热路径上
    零开销（gateway 的 chainTraceSink 直接返回 nil）
  - 事件是纯观测：scheduler 不基于它做任何分支，gateway 也不把它喂回路由/
    冷却/配额

四种 kind：tier_skip / slot_fail / tier_busy / selected，selected 每次成功
遍历恰好一次且是最后一步。顺序保证所有 step 在 routed 之前。

## 暴露给插件
新增 chain_step stage（逐个步骤），并在 request_end 载荷里加三个便于做报表的
字段：chain_walk（上限 12 步，防审计记录膨胀）、degraded、tier_served。

## ★ 计费口径（我按推荐的做，已写进文档，需要你确认）
**按实际服务的模型计费**：降级到 tier 3 仍按 tier 3 的价算，轨迹只作观测。
理由与 §7.5 的边界一致——插件只报表不执法，两套口径混在一起会引出"降级该不该
多收钱"这种无法从代码判断的争议。若要改成"按本该用的档计价"，需要在 models
价目里允许按 tier 定价，这我没做，因为那是个产品决策。

## 计费插件同步消费
by_tier_served / skip_reasons / degraded_reqs 三个新维度。skip_reasons 的等待
时长做了归一（`no free slot within <wait>`），否则 busy-wait 文案一变就多一行。
降级次数在 request_end 里计而不是在 chain_step 里计：一次降级的请求要走多步，
按步计会重复计数。

## 判据（346 个测试全绿，新增 15 个）
  scheduler  6 个：正常路径只发一个 selected / 跳档+降级可见 / 硬失败与跳档
                严格区分（不可混为一谈，否则抖动上游看起来像空闲上游）/
                nil sink 安全 / 全失败时轨迹与 ChainErr 并存且不互相破坏 /
                空链不发事件
  gateway    1 个端到端：tier 1 全 500 → 插件收到 slot_fail(tier 1) +
                selected(tier 2)，request_end 的 tier_served=2 且 degraded=true
  lua        2 个：降级计数与按实际模型计价 / 跳过原因归一聚合
  lua        1 个：chain_step 是真 stage 且顺序正确

3 个变异都红：去掉 slot_fail（3 个判据红）/ 去掉 tier_skip（1 个）/
去掉 degraded 字段（1 个）。
This commit is contained in:
JianFeeeee
2026-10-02 01:03:39 +08:00
parent 8c18e0c3d7
commit 42764bc99e
11 changed files with 766 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
internal/lua/billing_test.go
View File

@ -294,3 +294,85 @@ func TestBillingPluginLoadedByDefault(t *testing.T) {
t.Errorf("billing.lua was not written to the plugin dir: %v", err)
}
}
// TestBillingCountsDegradations: the plugin must distinguish a request that had
// to drop below the top tier from one the top tier served. Without the chain
// trace these were identical in the accounts, so a quietly degraded gateway
// looked healthy while spending more per request.
func TestBillingCountsDegradations(t *testing.T) {
ps, _ := billingVM(t)
_ = ps.SetState("billing", map[string]interface{}{
"prices": map[string]interface{}{
"models": map[string]interface{}{
"hi-tier": map[string]interface{}{"prompt": 1e-5, "completion": 1e-5},
"lo-tier": map[string]interface{}{"prompt": 1e-6, "completion": 1e-6},
},
},
})
// Request 1: degraded. tier 1 hard-failed, tier 2 served it.
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "slot_fail", "tier": 1, "source": "t1", "model": "hi-tier",
})
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "selected", "tier": 2, "source": "t2", "model": "lo-tier",
})
ps.Fire(StageRequestEnd, map[string]interface{}{
"model": "lo-tier", "source": "t2", "key": "***d1", "ok": true,
"prompt_tokens": 1000, "completion_tokens": 1000,
"degraded": true, "tier_served": 2, "time": 1750000000000,
})
// Request 2: clean, served by the top tier.
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "selected", "tier": 1, "source": "t1", "model": "hi-tier",
})
ps.Fire(StageRequestEnd, map[string]interface{}{
"model": "hi-tier", "source": "t1", "key": "***d1", "ok": true,
"prompt_tokens": 1000, "completion_tokens": 1000,
"degraded": false, "tier_served": 1, "time": 1750000000000,
})
st := stateOf(t, ps)
if got := st["degraded_reqs"].(float64); got != 1 {
t.Errorf("degraded_reqs = %v, want 1 (one of the two requests dropped a tier)", got)
}
tiers := st["by_tier_served"].(map[string]interface{})
if tiers["2"].(float64) != 1 {
t.Errorf("by_tier_served[2] = %v, want 1", tiers["2"])
}
if tiers["1"].(float64) != 1 {
t.Errorf("by_tier_served[1] = %v, want 1", tiers["1"])
}
// Cost reflects the model actually served, not the one that should have been.
// 1000*1e-6*2 = 0.002 for the degraded one, 1000*1e-5*2 = 0.02 for the clean one.
approx(t, "total", st["total"].(map[string]interface{})["cost"].(float64), 0.022)
}
// TestBillingAggregatesSkipReasons: skip reasons are the actionable diagnostic
// ("no schedulable slot (cooling or quota exhausted)"), so they must be
// counted. The wait time is normalised, otherwise a fresh row per request would
// appear whenever the busy-wait text varies.
func TestBillingAggregatesSkipReasons(t *testing.T) {
ps, _ := billingVM(t)
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "tier_skip", "tier": 1, "reason": "no schedulable slot (cooling or quota exhausted)",
})
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "tier_busy", "tier": 2, "reason": "no free slot within 2s",
})
ps.Fire(StageChainStep, map[string]interface{}{
"kind": "tier_busy", "tier": 3, "reason": "no free slot within 2.0001s",
})
st := stateOf(t, ps)
reasons := st["skip_reasons"].(map[string]interface{})
if len(reasons) != 2 {
t.Errorf("skip_reasons = %v, want 2 (the two variable waits must collapse to one)", reasons)
}
busy, ok := reasons["no free slot within <wait>"]
if !ok {
t.Errorf("busy reason missing; got %v", reasons)
} else if busy.(float64) != 2 {
t.Errorf("busy count = %v, want 2 (two different wait texts, one cause)", busy)
}
}

26
internal/lua/plugins.go
View File

@ -64,6 +64,31 @@ const (
// tier (AUTO tier, -1 for the direct path), stream.
StageRouted Stage = "routed"
// StageChainStep fires ONCE PER STEP of an AUTO chain walk, and only on the
// AUTO path (a direct request has no chain and therefore emits nothing).
//
// This exists because StageRouted cannot express degradation: it fires once,
// after the walk, with the slot that finally won. "tier 1 was cooling so we
// dropped to tier 3" and "tier 1 served it" were indistinguishable. That
// distinction is the whole point of a priority chain, and it is what an
// operator debugging "why did my expensive model not get used" needs.
//
// payload:
// kind "tier_skip" | "slot_fail" | "tier_busy" | "selected"
// tier the AUTO tier this step belongs to (1 = highest priority)
// source / model set for slot_fail and selected
// reason human-readable cause, for tier_skip and tier_busy
// error the underlying error text, for slot_fail
// attempt 1-based slot attempt within this walk
//
// Ordering: every step precedes StageRouted, and the "selected" step is the
// last one. A plugin accumulating the walk therefore has the full picture
// by the time request_end arrives.
//
// These events are OBSERVATION ONLY — see the accounting note in
// docs/plugins.md: nothing here feeds back into routing, cooldown or quota.
StageChainStep Stage = "chain_step"
// StageRequestEnd fires exactly once per request, after the client response
// has been produced (or after a failure was recorded). payload adds:
// source, model, ok, status, latency_ms, first_byte_ms, prompt_tokens,
@ -78,6 +103,7 @@ const (
// AllStages is the firing order, used by the docs and by the hook listing.
var AllStages = []Stage{
StageRequestStart,
StageChainStep,
StageRouted,
StageRequestEnd,
}

40
internal/lua/plugins/billing.lua
View File

@ -194,9 +194,43 @@ end
-- ---------- hooks ----------
plugin.hooks = {
-- chain_step gives the per-tier walk; request_end gives the final accounting.
-- Subscribing to chain_step is OPTIONAL here: the totals are driven by
-- request_end alone, and the degradation counters below are pure observation.
-- A gateway with thousands of requests can drop this hook to save the
-- per-step Lua call without losing a single billed request.
chain_step = "on_chain_step",
request_end = "on_request_end",
}
-- Tracks how often a request had to drop below the top tier, and which tier
-- actually served it. Without this, "tier 1 was cooling" and "tier 1 served it"
-- are indistinguishable in the accounts, and a quietly degraded gateway looks
-- exactly like a healthy one.
plugin.state.degraded_reqs = 0
plugin.state.by_tier_served = {}
plugin.state.skip_reasons = {}
function plugin.on_chain_step(payload)
if payload == nil then return nil end
local s = plugin.state
if s == nil then return nil end
if s.by_tier_served == nil then s.by_tier_served = {} end
if s.skip_reasons == nil then s.skip_reasons = {} end
if payload.kind == "selected" then
local t = tostring(payload.tier or "?")
s.by_tier_served[t] = (s.by_tier_served[t] or 0) + 1
elseif payload.kind == "tier_skip" or payload.kind == "tier_busy" then
-- reason text is the ACTIONABLE part; normalise the volatile bits so the
-- same cause aggregates instead of creating a new row per request.
local r = tostring(payload.reason or payload.kind or "unknown")
r = string.gsub(r, "within [%d%.%a]+", "within <wait>")
s.skip_reasons[r] = (s.skip_reasons[r] or 0) + 1
end
return nil
end
function plugin.on_request_end(payload)
if payload == nil then return nil end
local prompt = tonumber(payload.prompt_tokens) or 0
@ -216,6 +250,11 @@ function plugin.on_request_end(payload)
if s.by_key == nil then s.by_key = {} end
if s.by_day == nil then s.by_day = {} end
if s.started == nil then s.started = payload.time or 0 end
if s.degraded_reqs == nil then s.degraded_reqs = 0 end
-- Degradation is counted here rather than in the chain_step hook because
-- request_end sees the whole walk at once: one degraded request must count
-- once, whereas the walk may contain several skipped tiers.
if payload.degraded then s.degraded_reqs = s.degraded_reqs + 1 end
add(s.total, cost, prompt, completion, ok)
if payload.source ~= nil and payload.source ~= "" then
@ -313,6 +352,7 @@ plugin.ui = {
document.getElementById("billing-kpis").innerHTML = [
["Total", money(t.cost, cur)],
["Requests", t.requests || 0],
["Degraded", s.degraded_reqs || 0],
["Prompt tokens", t.prompt_tokens || 0],
["Completion tokens", t.completion_tokens || 0],
["Failures", t.failures || 0]

60
internal/lua/plugins_test.go
View File

@ -311,3 +311,63 @@ func TestPluginFireWithNoPluginsIsNoop(t *testing.T) {
t.Errorf("empty registry misbehaved: %+v", out)
}
}
// TestChainStepIsARealStage: chain_step is documented as a distinct stage that
// fires once per step of an AUTO walk. If it were only a field on `routed`, a
// plugin author following the docs would silently get one event instead of the
// whole walk.
func TestChainStepIsARealStage(t *testing.T) {
found := false
for _, s := range AllStages {
if s == StageChainStep {
found = true
}
}
if !found {
t.Fatal("StageChainStep is not in AllStages, so the dispatcher never registers it")
}
// Ordering: it must sit between request_start and routed, which is what
// docs/plugins.md promises.
var iStart, iStep, iRouted = -1, -1, -1
for i, s := range AllStages {
switch s {
case StageRequestStart:
iStart = i
case StageChainStep:
iStep = i
case StageRouted:
iRouted = i
}
}
if !(iStart < iStep && iStep < iRouted) {
t.Errorf("stage order = %v, want request_start < chain_step < routed", AllStages)
}
_, ps, _ := newPluginVM(t)
code := `
local p = { name = "stepper" }
p.hooks = { chain_step = "s" }
function p.s(payload)
payload.kinds = (payload.kinds or "")
return nil
end
return p
`
if err := loadPlugin(t, ps, "stepper", code); err != nil {
t.Fatalf("load: %v", err)
}
// It must actually dispatch.
seen := false
for _, row := range ps.List() {
if row["name"] == "stepper" {
hooks := row["hooks"].([]string)
for _, h := range hooks {
if h == string(StageChainStep) {
seen = true
}
}
}
}
if !seen {
t.Error("a plugin registered for chain_step is not reported as such")
}
}