mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 08:57:57 +00:00
feat: AUTO chain rewrite — silent failover+busy skip+pref round-robin+503 tier summary; chain edits reset slot cooldowns (P0/P1); stats by_status + audit jsonl rotation; UI priority-page health badges & status-code card; ctx-menu capture-phase close (outside-press guard); main.go ops warnings; local bundled-Lua verified tests (3 latent bugs fixed); plan.md
This commit is contained in:
JianFeeeee
@ -40,6 +40,7 @@ func newTestGateway(t *testing.T, srcs ...config.Source) *Gateway {
|
||||
cfg := &config.Config{
|
||||
AdapterDir: filepath.Join(t.TempDir(), "adapters"),
|
||||
RuntimeFile: filepath.Join(t.TempDir(), "runtime.json"),
|
||||
GatewayKeys: []string{"sk-test"},
|
||||
Sources: srcs,
|
||||
}
|
||||
if err := cfg.ApplyDefaults(); err != nil {
|
||||
@ -69,6 +70,192 @@ func doReq(t *testing.T, g *Gateway, method, path, body string) *httptest.Respon
|
||||
return rr
|
||||
}
|
||||
|
||||
// upstreamCtrl toggles a mocked upstream's behavior between requests.
|
||||
type upstreamCtrl struct {
|
||||
status int // 0 = healthy; else every request fails with that status
|
||||
hits int // chat call count
|
||||
}
|
||||
|
||||
// upstream returns a mocked OpenAI upstream driven by ctrl.status.
|
||||
func upstream(t *testing.T, ctrl *upstreamCtrl) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
ctrl.hits++
|
||||
if ctrl.status != 0 {
|
||||
w.WriteHeader(ctrl.status)
|
||||
fmt.Fprint(w, `{"error":"boom"}`)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"choices":[{"message":{"content":"pong"},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}`)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestChatAutoChainTierFailover: AUTO chain, first slot hard-fails, the pass
|
||||
// moves on within the same tier and the request is served by the next slot.
|
||||
func TestChatAutoChainTierFailover(t *testing.T) {
|
||||
a, b := &upstreamCtrl{status: 500}, &upstreamCtrl{}
|
||||
aUp := upstream(t, a)
|
||||
bUp := upstream(t, b)
|
||||
defer aUp.Close()
|
||||
defer bUp.Close()
|
||||
g := newTestGateway(t,
|
||||
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
||||
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m", Priority: 10}}},
|
||||
)
|
||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var cc ChatCompletion
|
||||
_ = json.Unmarshal(rr.Body.Bytes(), &cc)
|
||||
if cc.Model != "b-m" {
|
||||
t.Fatalf("AUTO served %q, want b-m", cc.Model)
|
||||
}
|
||||
if a.hits == 0 || b.hits == 0 {
|
||||
t.Fatalf("hit counts a=%d b=%d, want both > 0", a.hits, b.hits)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatAutoChain503Summary: every AUTO slot fails -> 503 whose message
|
||||
// names each failed tier/source/model.
|
||||
func TestChatAutoChain503Summary(t *testing.T) {
|
||||
a, b := &upstreamCtrl{status: 500}, &upstreamCtrl{status: 500}
|
||||
aUp := upstream(t, a)
|
||||
bUp := upstream(t, b)
|
||||
defer aUp.Close()
|
||||
defer bUp.Close()
|
||||
g := newTestGateway(t,
|
||||
config.Source{Name: "a", BaseURL: aUp.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
||||
config.Source{Name: "b", BaseURL: bUp.URL, Adapter: "openai", Models: []config.Model{{ID: "b-m", Priority: 10}}},
|
||||
)
|
||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "all auto tiers failed") ||
|
||||
!strings.Contains(rr.Body.String(), "a/a-m") ||
|
||||
!strings.Contains(rr.Body.String(), "b/b-m") {
|
||||
t.Fatalf("503 must summarize every tier, body=%s", rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
// TestChatAutoQuotaSkip: a slot whose token quota is exhausted is dropped
|
||||
// from scheduling; with no other slot the chain answers 503 naming the quota.
|
||||
func TestChatAutoQuotaSkip(t *testing.T) {
|
||||
ctrl := &upstreamCtrl{}
|
||||
up := upstream(t, ctrl)
|
||||
defer up.Close()
|
||||
g := newTestGateway(t,
|
||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
||||
)
|
||||
// one slot with an hourly quota of 1 token
|
||||
rr := doReq(t, g, "PUT", "/api/auto",
|
||||
`{"rules":[{"model":"a-m","tier":0,"token_quota":1,"period":"hour"}]}`)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("put auto status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
// first request consumes 4 tokens -> quota exhausted
|
||||
rr = doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("first status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
// second request must skip the exhausted slot and fail 503
|
||||
rr = doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("quota status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !strings.Contains(rr.Body.String(), "quota exhausted") {
|
||||
t.Fatalf("503 must name the quota reason, body=%s", rr.Body.String())
|
||||
}
|
||||
if ctrl.hits != 1 {
|
||||
t.Fatalf("upstream hits = %d, want 1 (exhausted slot must not be called)", ctrl.hits)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoStatesReportChainHealth: GET /api/auto reports per-slot health for
|
||||
// the priority-page UI; a chain edit resets the failure state to zero.
|
||||
func TestAutoStatesReportChainHealth(t *testing.T) {
|
||||
ctrl := &upstreamCtrl{status: 500}
|
||||
up := upstream(t, ctrl)
|
||||
defer up.Close()
|
||||
g := newTestGateway(t,
|
||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
||||
)
|
||||
doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
|
||||
fetch := func() []core.AutoSlotState {
|
||||
rr := doReq(t, g, "GET", "/api/auto", "")
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("get auto status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Rules []config.ModelScope `json:"rules"`
|
||||
States []core.AutoSlotState `json:"states"`
|
||||
}
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("unmarshal: %v", err)
|
||||
}
|
||||
return body.States
|
||||
}
|
||||
|
||||
st := fetch()
|
||||
if len(st) != 1 || st[0].Model != "a-m" || st[0].Source != "a" {
|
||||
t.Fatalf("want 1 slot a/a-m, got %#v", st)
|
||||
}
|
||||
if st[0].FailCount == 0 || !st[0].Cooling {
|
||||
t.Fatalf("slot must report the failure (fail=%d cooling=%v)", st[0].FailCount, st[0].Cooling)
|
||||
}
|
||||
|
||||
doReq(t, g, "PUT", "/api/auto",
|
||||
`{"rules":[{"model":"a-m","tier":0}]}`)
|
||||
st = fetch()
|
||||
if st[0].FailCount != 0 || st[0].Cooling {
|
||||
t.Fatalf("edit must reset health, got %#v", st[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestAutoSaveResetsCooldown: editing the AUTO chain clears the cooldown of
|
||||
// its slots, so a fixed upstream is schedulable again without waiting (P1).
|
||||
func TestAutoSaveResetsCooldown(t *testing.T) {
|
||||
ctrl := &upstreamCtrl{status: 500}
|
||||
up := upstream(t, ctrl)
|
||||
defer up.Close()
|
||||
g := newTestGateway(t,
|
||||
config.Source{Name: "a", BaseURL: up.URL, Adapter: "openai", Models: []config.Model{{ID: "a-m", Priority: 100}}},
|
||||
)
|
||||
rr := doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("expect 503 while upstream down, got %d", rr.Code)
|
||||
}
|
||||
p := g.core.ProviderForSlot("a-m", "a")
|
||||
if p == nil || p.ModelAvailable("a-m") {
|
||||
t.Fatal("a-m must be cooling after the failure")
|
||||
}
|
||||
// editing the chain (same rules) must clear the cooldown immediately
|
||||
rr = doReq(t, g, "PUT", "/api/auto",
|
||||
`{"rules":[{"model":"a-m","tier":0}]}`)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("put auto status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !p.ModelAvailable("a-m") {
|
||||
t.Fatal("SaveAutoRules must reset the slot cooldown")
|
||||
}
|
||||
// healed upstream -> AUTO serves again on the next request
|
||||
ctrl.status = 0
|
||||
rr = doReq(t, g, "POST", "/v1/chat/completions",
|
||||
`{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`)
|
||||
if rr.Code != 200 {
|
||||
t.Fatalf("AUTO after reset status=%d body=%s", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestChatSingle(t *testing.T) {
|
||||
up := mockUpstream()
|
||||
defer up.Close()
|
||||
@ -446,4 +633,4 @@ func TestAPIChatInternal(t *testing.T) {
|
||||
if !strings.Contains(rr.Body.String(), "pong") {
|
||||
t.Fatalf("api chat body=%s", rr.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user