fix(scheduler): rebalance the pref score so metered sources aren't written off silently

Report: an allowance-metered source (sensenova) had its deepseek model sitting at
pref -10 while the WebUI showed zero failures. All three numbers were accurate,
and they exposed three compounding problems.

1. Quota exhaustion cost the SAME score as a real failure. RecordQuotaExhausted
   intentionally avoids failCount (an exhausted allowance is not a fault), so the
   UI showed fails=0 / cooling=false — yet it deducted the full prefFailStep (5).
   For a metered source, running out of budget is an everyday event, so the score
   drifted deep negative with no visible cause. Now quota and 429 events cost
   prefQuotaStep (1): the cooldown already keeps the slot out of rotation until
   the window resets, the score only needs a mild preference for slots with budget.

2. Recovery was 5:1 asymmetric. A failure cost -5 but a success only +1, so a
   slot at -10 needed ten consecutive successes just to reach neutral — which it
   could never get, because a low score makes the scheduler not pick it in the
   first place (starvation). Success now rewards prefSuccessStep (2): recovery
   from -10 needs five successes, while a real failure still outweighs one.

3. No idle decay. A penalised slot kept its negative score forever once it stopped
   being selected. Pref() now applies lazy decay: after prefDecayAfter (2 min) of
   no outcome, the score drifts one step back toward 0 per interval (never past
   0, never touches positive scores). Applied in Pref() and TryProbe(), so a
   naturally-recovered idle slot is schedulable again without needing a probe.

The failure penalty itself is unchanged (prefFailStep=5), so genuinely broken
upstreams are still marked as clearly worse than healthy ones.

Tests: quota penalty lighter than failure; 5 quota resets stay well above the
floor with failCount untouched; 429 is quota-class; recovery from -10 needs <=5;
idle decay rehabilitates a written-off slot, stops at 0, and never drags a
positive score; decay repeatedly lifts a slot off the prefMin floor. Updated the
two pre-existing tests that asserted the old -5/129 +1 values.
This commit is contained in:
JianFeeeee
2026-08-31 11:50:47 +08:00
parent 2e3d5b79ad
commit 5b628b4d6f
2 changed files with 250 additions and 21 deletions

View File

@ -242,8 +242,10 @@ func TestModelStateCooldownAndRecovery(t *testing.T) {
if st.FailCount() != 0 {
t.Fatalf("fail count after success = %d", st.FailCount())
}
if st.Pref() != 1-int64(prefFailStep) {
t.Fatalf("pref after one failure (-5) then success (+1) = %d, want %d", st.Pref(), 1-int64(prefFailStep))
// failure penalties prefFailStep, success rewards prefSuccessStep:
// -5 then +2 = -3
if st.Pref() != int64(prefSuccessStep)-int64(prefFailStep) {
t.Fatalf("pref after one failure then success = %d, want %d", st.Pref(), int64(prefSuccessStep)-int64(prefFailStep))
}
}
@ -265,8 +267,8 @@ func TestModelStateRateLimitShortCooldown(t *testing.T) {
if st.FailCount() != 15 {
t.Fatalf("fail count = %d, want 15 (counted but not escalating)", st.FailCount())
}
if st.Pref() != -15*int64(prefFailStep) && st.Pref() > int64(prefMin) {
t.Fatalf("pref = %d", st.Pref())
if st.Pref() != -15*int64(prefQuotaStep) {
t.Fatalf("pref after 15 x 429 = %d, want %d (a 429 is a quota-class event, not a fault)", st.Pref(), -15*int64(prefQuotaStep))
}
if p.ModelAvailable("m") {
t.Fatal("model must be cooling right after a 429")
@ -1044,3 +1046,140 @@ func TestRecoveryProbeBudgetIsBounded(t *testing.T) {
t.Fatalf("3 probe windows must cost exactly 3 upstream requests, got %d", extra)
}
}
// TestQuotaPenaltyIsLighterThanFailure is the regression for the report
// "今天一次失败没看到,分数目前已经 -10 了" on an allowance-metered source
// (sensenova). RecordQuotaExhausted deliberately does not touch failCount — the
// WebUI therefore showed fails=0 and cooling=false — yet it used to charge the
// SAME score penalty as a real failure, so two ordinary quota resets dragged the
// slot to -10 with no visible fault. An exhausted allowance means "no budget
// right now", not "this model is broken": the cooldown already keeps it out of
// rotation, so the score penalty must be mild.
func TestQuotaPenaltyIsLighterThanFailure(t *testing.T) {
quota := &ModelState{}
quota.RecordQuotaExhausted(time.Minute)
fail := &ModelState{}
fail.RecordFailure(false)
qp, fp := quota.Pref(), fail.Pref()
if qp <= fp {
t.Fatalf("a quota reset must cost less score than a real failure: quota=%d failure=%d", qp, fp)
}
if qp != -prefQuotaStep {
t.Errorf("quota penalty = %d, want -%d", qp, prefQuotaStep)
}
if fp != -prefFailStep {
t.Errorf("failure penalty = %d, want -%d", fp, prefFailStep)
}
// The reported symptom: several quota resets in a day must NOT approach the
// prefMin floor, since each one is a normal event for a metered source.
s := &ModelState{}
for i := 0; i < 5; i++ {
s.RecordQuotaExhausted(time.Minute)
}
if got := s.Pref(); got <= prefMin/2 {
t.Errorf("5 quota resets sank the score to %d (floor %d); a metered source would be written off", got, prefMin)
}
if s.FailCount() != 0 {
t.Errorf("quota exhaustion must not inflate failCount, got %d", s.FailCount())
}
}
// TestRateLimitPenaltyIsMild covers the sibling case: a 429 means "too fast",
// not "broken", so a popular-but-healthy slot must not sink for being throttled.
func TestRateLimitPenaltyIsMild(t *testing.T) {
s := &ModelState{}
s.RecordRateLimit()
if got := s.Pref(); got != -prefQuotaStep {
t.Errorf("429 penalty = %d, want -%d (same class as a quota event)", got, prefQuotaStep)
}
}
// TestSuccessRecoversFasterThanBefore pins the penalty/reward asymmetry. Reward
// used to be +1 against a -5 penalty, so a slot at -10 needed TEN consecutive
// successes just to reach neutral — in practice it never got there, because a
// low score makes the slot unattractive to the scheduler in the first place.
func TestSuccessRecoversFasterThanBefore(t *testing.T) {
s := &ModelState{}
s.RecordFailure(false)
s.RecordFailure(false) // -10, the score from the bug report
n := 0
for s.Pref() < 0 {
s.RecordSuccess()
n++
if n > 50 {
t.Fatal("score never recovered")
}
}
// with +2 per success, -10 needs 5; the old +1 needed 10
if n > 5 {
t.Errorf("recovery from -10 took %d successes, want <= 5", n)
}
// a real failure must still outweigh a single success, or failures stop
// meaning anything
if prefSuccessStep >= prefFailStep {
t.Errorf("success reward %d must stay below the failure penalty %d", prefSuccessStep, prefFailStep)
}
}
// TestPrefDecayRehabilitatesIdleSlot closes the starvation loop: a penalised
// slot is unattractive, so the scheduler stops picking it, so it never earns the
// successes that would rehabilitate it. Without decay the score stays negative
// forever even though the upstream may be perfectly healthy.
func TestPrefDecayRehabilitatesIdleSlot(t *testing.T) {
s := &ModelState{}
s.RecordFailure(false)
s.RecordFailure(false)
if s.Pref() != -2*prefFailStep {
t.Fatalf("setup: pref=%d", s.Pref())
}
// backdate the last touch by three decay intervals
step := int64(prefDecayAfter / time.Second)
s.prefTouched.Store(time.Now().Unix() - 3*step)
got := s.Pref()
if got != -2*prefFailStep+3 {
t.Errorf("after 3 idle intervals pref=%d, want %d", got, -2*prefFailStep+3)
}
// decay must never overshoot into positive territory
s.prefTouched.Store(time.Now().Unix() - 1000*step)
if got := s.Pref(); got != 0 {
t.Errorf("decay overshot to %d, want to stop at 0", got)
}
// a positive score must not be dragged down for being idle
good := &ModelState{}
good.RecordSuccess()
good.RecordSuccess()
before := good.Pref()
good.prefTouched.Store(time.Now().Unix() - 1000*step)
if after := good.Pref(); after != before {
t.Errorf("idle decay must not touch a positive score: %d -> %d", before, after)
}
}
// TestDecayLiftsSlotOffTheFloor verifies decay actually restores schedulability:
// a slot at prefMin is refused by ModelAvailable, and before decay existed only
// a probe could rescue it.
func TestDecayLiftsSlotOffTheFloor(t *testing.T) {
s := &ModelState{}
for i := 0; i < 10; i++ {
s.RecordFailure(false)
}
if s.Pref() != prefMin {
t.Fatalf("setup: want floor %d, got %d", prefMin, s.Pref())
}
// clear the cooldown so only the score gates the slot
s.cooldownUntil.Store(0)
s.cooldownFrom.Store(0)
step := int64(prefDecayAfter / time.Second)
s.prefTouched.Store(time.Now().Unix() - 2*step)
if got := s.Pref(); got <= prefMin {
t.Errorf("decay failed to lift the slot off the floor: %d", got)
}
}