jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-11-12 20:27:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(quota): 未就绪适配器污染负 memo；被拒凭据被后续 404 掩盖

Browse Source 
端到端验证时撞出两个真缺陷，都是单元测试全绿、隔离实例打真上游才暴露。

一、负 memo 语义混用

负 memo 说的是「这台 HOST 没有余额端点」，且无条件在每次探测前生效。
openDesign 缺 control_key 说的是「这台部署还没配」—— 两件事共用一个条目：

  1. 未配 control_key → 探测 → 负 memo
  2. 操作员写入 key   → 探测被退避拦下，adapter 根本没跑
  3. 结果：修复看起来无效，源持续沉默（退避窗口可达数小时）

不能靠「skip 就不记 memo」解决：真实 vela 主机上另外三个 adapter 都返回
404（关于主机的持久结论，正是 memo 存在的理由），verdict 一定是普通 skip。
做法是 memo 条目带上「哪些 vendor 因自身未配置而无法判断」：
  - 未就绪的 adapter 直接不跑（跑了只能得到 401 或伪造的 skip）
  - 条目带 unready 集合；该 vendor 配置好后条目作废重算
  - 仍缺配置时条目依然写入（否则无看板的主机回到每次开页 60 秒超时），
    但不退避 —— 退避会拒绝那个本来能成功的探测

二、401 被下一个 adapter 的 404 掩盖

adapter 循环不停在 error，而「最后一个胜」不精确：openDesign 走厂商的整份
主机列表，是最可能看到「key rejected」的那个，但它也可能排在最前（memo 记
得它应答过，下次就优先它）—— 于是它的 401 被下一个 adapter 的
404「no dashboard here」分类成「这台主机没有余额端点」。两种情况的修复动作
完全相反，输出却一样。规则改为：被拒凭据优先，且不被覆盖。

判据：12/12 变异全杀。三条曾经存活或误杀，逐条查清：
  - rejected-as-skip 存活是因为测试没有复现生产调用顺序（memo 空 → openDesign
    排最后 → 旧规则也能过），测试已改为预置 memo 并让另三个 adapter 走同一个
    stub（它们读 base 而非 api_base，否则会全部传输失败）
  - no-cred-guard 并非存活，只是杀它的不是我以为的那个测试（门禁后面还有
    一道检查），预期已改正
  - unready-memo-not-dropped 是真等价变异（known=false 与 nil 只差一次廉价
    探活），已从清单移除
This commit is contained in:
JianFeeeee
2026-10-07 01:42:24 +08:00
parent 65075b6d44
commit 83f1ec0807
3 changed files with 390 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
internal/lua/quota_opendesign_test.go
View File

@ -317,34 +317,66 @@ func TestQuotaOpenDesignHasNoFabricatedPeriodWindow(t *testing.T) {
// means the credential is wrong — a fact the operator must fix — so it must not
// be filed under "skip" alongside "this vendor has no balance endpoint". The two
// produce the same blank card but wildly different next actions.
//
// Two details make the test actually exercise the rule instead of passing by
// accident, and both were found by a mutation surviving:
//
// 1. The memo is pre-seeded so openDesign runs FIRST. The memo remembers which
// vendor answered last and that vendor is tried first; once a host has ever
// answered through openDesign it runs first forever after. With the default
// order (openDesign last) its 401 IS the last verdict, so even a plain
// "last error wins" rule passes.
// 2. The other three adapters must ANSWER (404), not fail. They read `base`, not
// `api_base`, so the source base_url points at the same stub — otherwise they
// all error out on an unreachable host and the final verdict is an error
// regardless of the rule being tested.
func TestQuotaOpenDesignRejectedControlKeyIsAnError(t *testing.T) {
srv := odServer(t, nil, `{"error":"unauthenticated"}`, `{"error":"unauthenticated"}`,
http.StatusUnauthorized, http.StatusUnauthorized)
defer srv.Close()
ps, _ := odPlugins(t)
// Host key must match what hostOf(base) yields for the source below.
src := map[string]interface{}{
"name": "vela", "base_url": srv.URL, "api_key": "k",
}
if err := ps.SetState("quota", map[string]interface{}{
"prices": map[string]interface{}{
"control_key": "vela_ctrl_WRONG",
"api_base": srv.URL + "/api/v1",
},
// Flat, NOT nested under a "state" key: SetState pulls `prices` out and
// stores the REST of the payload as the plugin's state. Nesting it would
// store {state: {probe_memo: ...}} and the memo would never be read.
"probe_memo": map[string]interface{}{
hostOnly(srv.URL): "opendesign",
},
}); err != nil {
t.Fatal(err)
}
rep := collectOpenDesign(t, ps, "https://amr-link.open-design.ai/v1")
out, err := ps.Call("quota", "collect_source", map[string]interface{}{
"name": "vela", "sources": []interface{}{src},
})
if err != nil {
t.Fatal(err)
}
rep, _ := out.(map[string]interface{})
if rep == nil {
t.Fatal("no report produced")
}
if rep["vendor"] == "opendesign" {
t.Fatalf("a rejected control key must not produce a balance: %v", rep)
}
// The plugin reports a credential failure as kind="error", which the page
// renders as a red "读取失败" row. Filing it as a skip instead would show the
// same blank card as "this vendor has no balance endpoint" — and send the
// operator to investigate the vendor rather than re-run the login.
// The plugin reports a credential failure as an error, which the page renders
// as a red "读取失败" row. Filing it as a skip instead shows the same blank
// card as "this vendor has no balance endpoint" — and sends the operator to
// investigate the vendor rather than re-run the login.
if rep["skipped"] == true {
t.Fatalf("a rejected control key was reported as a SKIP: %v — the two "+
"require different fixes and must not look alike", rep)
t.Fatalf("a rejected control key was reported as a SKIP: %v — the other three "+
"adapters answered 404 for this host, so the final verdict is a skip unless "+
"a rejected credential outranks it. The operator would go looking at the "+
"vendor instead of re-running the login.", rep)
}
msg, _ := rep["error"].(string)
if !strings.Contains(msg, "rejected") {
@ -404,48 +436,133 @@ func TestQuotaOpenDesignDoesNotDefaultMissingRate(t *testing.T) {
// working. Without it the adapter probes the inference host, which serves no
// billing route at all, and every OpenDesign source silently skips.
//
// Asserted BEHAVIOURALLY, through the outbound allowlist rather than on the
// source text. The allowlist in these tests holds only 127.0.0.1, so a request
// to any real host is refused with a message that NAMES the host it tried. That
// makes the refusal the observation: "amr-api.open-design.ai" in the reason
// proves the derivation ran, and its absence proves it did not. The earlier
// version of this test grepped quota.lua for the gsub pattern, which pins the
// spelling of the rule instead of its effect — it went red the moment the rule
// was tightened to a full-hostname match, while the behaviour was unchanged.
// Asserted on the exported derivation rather than on the source text or on a
// failed probe's error message. The source-text version pinned the SPELLING of
// the rule: it went red the moment the rule was tightened from an `amr-link.`
// prefix to a full hostname, while the behaviour was unchanged. The
// error-message version inferred "which host did we ask?" from an allowlist
// refusal, which stops working the moment any adapter earlier in the chain also
// fails — and one of them always does.
func TestQuotaOpenDesignHostRewritingIsUsed(t *testing.T) {
ps, _ := odPlugins(t)
// No api_base: the adapter must derive the billing host itself. The control
// key IS set so the credential guard is not what stops us.
cases := []struct{ in, want string }{
// The real production shape: base_url carries a path.
{"https://amr-link.open-design.ai/v1", "https://amr-api.open-design.ai/api/v1"},
// And without one, so the derivation cannot depend on the path being there.
{"https://amr-link.open-design.ai", "https://amr-api.open-design.ai/api/v1"},
// A trailing slash must not produce a double slash.
{"https://amr-link.open-design.ai/v1/", "https://amr-api.open-design.ai/api/v1"},
}
for _, c := range cases {
out, err := ps.Call("quota", "resolve_api_base", map[string]interface{}{
"base_url": c.in,
})
if err != nil {
t.Fatalf("resolve_api_base(%q): %v", c.in, err)
}
rep, _ := out.(map[string]interface{})
if got, _ := rep["api_base"].(string); got != c.want {
t.Errorf("resolve_api_base(%q) = %q, want %q — the probe would go to the "+
"inference host, which serves no billing route, and the source would "+
"silently stay unlisted", c.in, got, c.want)
}
}
// A lookalike host must NOT be rewritten into an OpenDesign address: an
// `amr-link.` prefix rule would turn amr-link.example.com into amr-api...,
// sending an unrelated provider's probe to a host that is not its own.
out, err := ps.Call("quota", "resolve_api_base", map[string]interface{}{
"base_url": "https://amr-link.example.com/v1",
})
if err != nil {
t.Fatal(err)
}
rep, _ := out.(map[string]interface{})
if got, _ := rep["api_base"].(string); got != "https://amr-link.example.com/api/v1" {
t.Errorf("a lookalike host was rewritten to %q", got)
}
// An explicit override wins (and is trimmed), so an operator can point at a
// moved host without waiting for a code change.
out, err = ps.Call("quota", "resolve_api_base", map[string]interface{}{
"base_url": "https://amr-link.open-design.ai/v1",
"api_base": "https://amr-api.example.net/api/v1/",
})
if err != nil {
t.Fatal(err)
}
rep, _ = out.(map[string]interface{})
if got, _ := rep["api_base"].(string); got != "https://amr-api.example.net/api/v1" {
t.Errorf("api_base override not honoured (or not trimmed): %q", got)
}
}
// TestQuotaUnreadyAdapterDoesNotSuppressAConfiguredHost is the end-to-end half of
// the unready rule, and it reproduces a bug seen on the live plugin.
//
// The negative memo means "this HOST exposes no balance endpoint" and is applied
// UNCONDITIONALLY, before any adapter runs. A host judged while openDesign had no
// prices.control_key therefore got memoised as having nothing and stayed
// suppressed after the key was supplied — observed live: unset key → memo → key
// set → probe skipped without a single request, so the fix the operator had just
// applied looked like it did nothing.
//
// It is NOT fixable by "do not memoise a skip": on a real OpenDesign host the
// other three adapters answer 404 ("no dashboard here"), which is durable and
// worth remembering, so the verdict would be recorded either way. The memo has to
// carry WHICH vendors were unable to judge the host, and the read side has to
// retire that record once they can.
func TestQuotaUnreadyAdapterDoesNotSuppressAConfiguredHost(t *testing.T) {
var seenAuth []string
srv := odServer(t, &seenAuth, odWallet, odSummary, 200, 200)
defer srv.Close()
ps, _ := odPlugins(t)
args := map[string]interface{}{
"name": "vela",
"sources": []interface{}{map[string]interface{}{
"name": "vela", "base_url": srv.URL + "/v1", "api_key": "k",
}},
}
// Step 1: no control_key. openDesign cannot judge the host, and no billing
// request goes anywhere.
out, err := ps.Call("quota", "collect_source", args)
if err != nil {
t.Fatal(err)
}
rep, _ := out.(map[string]interface{})
if rep["skipped"] != true {
t.Fatalf("expected a skip with no control key configured: %v", rep)
}
if len(seenAuth) != 0 {
t.Fatalf("an unconfigured adapter made a request: %q", seenAuth)
}
// Step 2: configure the key. The very next probe must reach the upstream. It
// is inside the backoff window (minutes of wall clock) while this runs in
// microseconds — so a host refused now is refused for the whole window in
// production, which is the bug, not a timing accident.
if err := ps.SetState("quota", map[string]interface{}{
"prices": map[string]interface{}{"control_key": "vela_ctrl_TESTKEY"},
"prices": map[string]interface{}{
"control_key": "vela_ctrl_TESTKEY",
"api_base": srv.URL + "/api/v1",
},
}); err != nil {
t.Fatal(err)
}
rep := collectOpenDesign(t, ps, "https://amr-link.open-design.ai/v1")
if rep == nil {
t.Fatal("no report produced")
out, err = ps.Call("quota", "collect_source", args)
if err != nil {
t.Fatal(err)
}
// The outbound allowlist in these tests holds only 127.0.0.1, so the request
// is refused before it leaves the process and the refusal NAMES the host it
// tried. The observation is therefore the refusal text, not the record as a
// whole: `host` is `hostOf(base)` — the SOURCE host, which is amr-link by
// design and says nothing about where the request went.
var msg string
if s, ok := rep["error"].(string); ok {
msg = s
rep, _ = out.(map[string]interface{})
if rep["vendor"] != "opendesign" || rep["skipped"] == true {
t.Fatalf("expected an opendesign balance once configured, got %v — the host "+
"is still suppressed by the provisional memo it earned while the adapter "+
"had no credential", rep)
}
if s, ok := rep["reason"].(string); ok {
msg += " " + s
}
if !strings.Contains(msg, "amr-api.open-design.ai") {
t.Fatalf("no evidence the adapter derived the billing host from the "+
"inference one: %q. Expected the allowlist refusal to name "+
"amr-api.open-design.ai, since that is the host the request must go to. "+
"(report: %v)", msg, rep)
}
if strings.Contains(msg, "amr-link") {
t.Fatalf("the adapter probed the INFERENCE host — it serves no billing "+
"route, so every OpenDesign source would silently skip: %q", msg)
if len(seenAuth) == 0 {
t.Fatal("no request reached the billing endpoint after the control key was " +
"supplied")
}
}