mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 00:48:00 +00:00
fix(scheduler): rebalance the pref score so metered sources aren't written off silently
Report: an allowance-metered source (sensenova) had its deepseek model sitting at pref -10 while the WebUI showed zero failures. All three numbers were accurate, and they exposed three compounding problems. 1. Quota exhaustion cost the SAME score as a real failure. RecordQuotaExhausted intentionally avoids failCount (an exhausted allowance is not a fault), so the UI showed fails=0 / cooling=false — yet it deducted the full prefFailStep (5). For a metered source, running out of budget is an everyday event, so the score drifted deep negative with no visible cause. Now quota and 429 events cost prefQuotaStep (1): the cooldown already keeps the slot out of rotation until the window resets, the score only needs a mild preference for slots with budget. 2. Recovery was 5:1 asymmetric. A failure cost -5 but a success only +1, so a slot at -10 needed ten consecutive successes just to reach neutral — which it could never get, because a low score makes the scheduler not pick it in the first place (starvation). Success now rewards prefSuccessStep (2): recovery from -10 needs five successes, while a real failure still outweighs one. 3. No idle decay. A penalised slot kept its negative score forever once it stopped being selected. Pref() now applies lazy decay: after prefDecayAfter (2 min) of no outcome, the score drifts one step back toward 0 per interval (never past 0, never touches positive scores). Applied in Pref() and TryProbe(), so a naturally-recovered idle slot is schedulable again without needing a probe. The failure penalty itself is unchanged (prefFailStep=5), so genuinely broken upstreams are still marked as clearly worse than healthy ones. Tests: quota penalty lighter than failure; 5 quota resets stay well above the floor with failCount untouched; 429 is quota-class; recovery from -10 needs <=5; idle decay rehabilitates a written-off slot, stops at 0, and never drags a positive score; decay repeatedly lifts a slot off the prefMin floor. Updated the two pre-existing tests that asserted the old -5/129 +1 values.
This commit is contained in:
@ -242,8 +242,10 @@ func TestModelStateCooldownAndRecovery(t *testing.T) {
|
||||
if st.FailCount() != 0 {
|
||||
t.Fatalf("fail count after success = %d", st.FailCount())
|
||||
}
|
||||
if st.Pref() != 1-int64(prefFailStep) {
|
||||
t.Fatalf("pref after one failure (-5) then success (+1) = %d, want %d", st.Pref(), 1-int64(prefFailStep))
|
||||
// failure penalties prefFailStep, success rewards prefSuccessStep:
|
||||
// -5 then +2 = -3
|
||||
if st.Pref() != int64(prefSuccessStep)-int64(prefFailStep) {
|
||||
t.Fatalf("pref after one failure then success = %d, want %d", st.Pref(), int64(prefSuccessStep)-int64(prefFailStep))
|
||||
}
|
||||
}
|
||||
|
||||
@ -265,8 +267,8 @@ func TestModelStateRateLimitShortCooldown(t *testing.T) {
|
||||
if st.FailCount() != 15 {
|
||||
t.Fatalf("fail count = %d, want 15 (counted but not escalating)", st.FailCount())
|
||||
}
|
||||
if st.Pref() != -15*int64(prefFailStep) && st.Pref() > int64(prefMin) {
|
||||
t.Fatalf("pref = %d", st.Pref())
|
||||
if st.Pref() != -15*int64(prefQuotaStep) {
|
||||
t.Fatalf("pref after 15 x 429 = %d, want %d (a 429 is a quota-class event, not a fault)", st.Pref(), -15*int64(prefQuotaStep))
|
||||
}
|
||||
if p.ModelAvailable("m") {
|
||||
t.Fatal("model must be cooling right after a 429")
|
||||
@ -1044,3 +1046,140 @@ func TestRecoveryProbeBudgetIsBounded(t *testing.T) {
|
||||
t.Fatalf("3 probe windows must cost exactly 3 upstream requests, got %d", extra)
|
||||
}
|
||||
}
|
||||
|
||||
// TestQuotaPenaltyIsLighterThanFailure is the regression for the report
|
||||
// "今天一次失败没看到,分数目前已经 -10 了" on an allowance-metered source
|
||||
// (sensenova). RecordQuotaExhausted deliberately does not touch failCount — the
|
||||
// WebUI therefore showed fails=0 and cooling=false — yet it used to charge the
|
||||
// SAME score penalty as a real failure, so two ordinary quota resets dragged the
|
||||
// slot to -10 with no visible fault. An exhausted allowance means "no budget
|
||||
// right now", not "this model is broken": the cooldown already keeps it out of
|
||||
// rotation, so the score penalty must be mild.
|
||||
func TestQuotaPenaltyIsLighterThanFailure(t *testing.T) {
|
||||
quota := &ModelState{}
|
||||
quota.RecordQuotaExhausted(time.Minute)
|
||||
fail := &ModelState{}
|
||||
fail.RecordFailure(false)
|
||||
|
||||
qp, fp := quota.Pref(), fail.Pref()
|
||||
if qp <= fp {
|
||||
t.Fatalf("a quota reset must cost less score than a real failure: quota=%d failure=%d", qp, fp)
|
||||
}
|
||||
if qp != -prefQuotaStep {
|
||||
t.Errorf("quota penalty = %d, want -%d", qp, prefQuotaStep)
|
||||
}
|
||||
if fp != -prefFailStep {
|
||||
t.Errorf("failure penalty = %d, want -%d", fp, prefFailStep)
|
||||
}
|
||||
|
||||
// The reported symptom: several quota resets in a day must NOT approach the
|
||||
// prefMin floor, since each one is a normal event for a metered source.
|
||||
s := &ModelState{}
|
||||
for i := 0; i < 5; i++ {
|
||||
s.RecordQuotaExhausted(time.Minute)
|
||||
}
|
||||
if got := s.Pref(); got <= prefMin/2 {
|
||||
t.Errorf("5 quota resets sank the score to %d (floor %d); a metered source would be written off", got, prefMin)
|
||||
}
|
||||
if s.FailCount() != 0 {
|
||||
t.Errorf("quota exhaustion must not inflate failCount, got %d", s.FailCount())
|
||||
}
|
||||
}
|
||||
|
||||
// TestRateLimitPenaltyIsMild covers the sibling case: a 429 means "too fast",
|
||||
// not "broken", so a popular-but-healthy slot must not sink for being throttled.
|
||||
func TestRateLimitPenaltyIsMild(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordRateLimit()
|
||||
if got := s.Pref(); got != -prefQuotaStep {
|
||||
t.Errorf("429 penalty = %d, want -%d (same class as a quota event)", got, prefQuotaStep)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSuccessRecoversFasterThanBefore pins the penalty/reward asymmetry. Reward
|
||||
// used to be +1 against a -5 penalty, so a slot at -10 needed TEN consecutive
|
||||
// successes just to reach neutral — in practice it never got there, because a
|
||||
// low score makes the slot unattractive to the scheduler in the first place.
|
||||
func TestSuccessRecoversFasterThanBefore(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordFailure(false)
|
||||
s.RecordFailure(false) // -10, the score from the bug report
|
||||
|
||||
n := 0
|
||||
for s.Pref() < 0 {
|
||||
s.RecordSuccess()
|
||||
n++
|
||||
if n > 50 {
|
||||
t.Fatal("score never recovered")
|
||||
}
|
||||
}
|
||||
// with +2 per success, -10 needs 5; the old +1 needed 10
|
||||
if n > 5 {
|
||||
t.Errorf("recovery from -10 took %d successes, want <= 5", n)
|
||||
}
|
||||
// a real failure must still outweigh a single success, or failures stop
|
||||
// meaning anything
|
||||
if prefSuccessStep >= prefFailStep {
|
||||
t.Errorf("success reward %d must stay below the failure penalty %d", prefSuccessStep, prefFailStep)
|
||||
}
|
||||
}
|
||||
|
||||
// TestPrefDecayRehabilitatesIdleSlot closes the starvation loop: a penalised
|
||||
// slot is unattractive, so the scheduler stops picking it, so it never earns the
|
||||
// successes that would rehabilitate it. Without decay the score stays negative
|
||||
// forever even though the upstream may be perfectly healthy.
|
||||
func TestPrefDecayRehabilitatesIdleSlot(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
s.RecordFailure(false)
|
||||
s.RecordFailure(false)
|
||||
if s.Pref() != -2*prefFailStep {
|
||||
t.Fatalf("setup: pref=%d", s.Pref())
|
||||
}
|
||||
|
||||
// backdate the last touch by three decay intervals
|
||||
step := int64(prefDecayAfter / time.Second)
|
||||
s.prefTouched.Store(time.Now().Unix() - 3*step)
|
||||
|
||||
got := s.Pref()
|
||||
if got != -2*prefFailStep+3 {
|
||||
t.Errorf("after 3 idle intervals pref=%d, want %d", got, -2*prefFailStep+3)
|
||||
}
|
||||
|
||||
// decay must never overshoot into positive territory
|
||||
s.prefTouched.Store(time.Now().Unix() - 1000*step)
|
||||
if got := s.Pref(); got != 0 {
|
||||
t.Errorf("decay overshot to %d, want to stop at 0", got)
|
||||
}
|
||||
|
||||
// a positive score must not be dragged down for being idle
|
||||
good := &ModelState{}
|
||||
good.RecordSuccess()
|
||||
good.RecordSuccess()
|
||||
before := good.Pref()
|
||||
good.prefTouched.Store(time.Now().Unix() - 1000*step)
|
||||
if after := good.Pref(); after != before {
|
||||
t.Errorf("idle decay must not touch a positive score: %d -> %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecayLiftsSlotOffTheFloor verifies decay actually restores schedulability:
|
||||
// a slot at prefMin is refused by ModelAvailable, and before decay existed only
|
||||
// a probe could rescue it.
|
||||
func TestDecayLiftsSlotOffTheFloor(t *testing.T) {
|
||||
s := &ModelState{}
|
||||
for i := 0; i < 10; i++ {
|
||||
s.RecordFailure(false)
|
||||
}
|
||||
if s.Pref() != prefMin {
|
||||
t.Fatalf("setup: want floor %d, got %d", prefMin, s.Pref())
|
||||
}
|
||||
// clear the cooldown so only the score gates the slot
|
||||
s.cooldownUntil.Store(0)
|
||||
s.cooldownFrom.Store(0)
|
||||
|
||||
step := int64(prefDecayAfter / time.Second)
|
||||
s.prefTouched.Store(time.Now().Unix() - 2*step)
|
||||
if got := s.Pref(); got <= prefMin {
|
||||
t.Errorf("decay failed to lift the slot off the floor: %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user