mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-21 01:17:59 +00:00
feat: proactive rate limiting (RPM) + 429 short cooldown for sources
- config.go: Source add RPM field (requests-per-minute cap, 0=unlimited) - provider.go: RecordRateLimit() — 429 uses fixed 30s cooldown, not exponential - provider.go: Throttle() — token-bucket proactive rate limiter, spaces requests at 60s/RPM interval, respects context cancellation - provider.go: ReportStatus() — 429 -> RecordRateLimit, 5xx -> RecordFailure - provider.go: Chat/ChatStream — wire Throttle after TryAcquire - api.go: sourcePayload + RPM, buildSource passes RPM through - ui/index.html: add RPM input field in source editor, bilingual i18n labels - deploy.sh: backup old binary + rollback on healthcheck failure - provider_test.go: TestModelStateRateLimitShortCooldown, TestThrottleSpacingAndCancel - config.yaml: sensenova rpm: 12
This commit is contained in:
@ -245,6 +245,77 @@ func TestModelStateCooldownAndRecovery(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelStateRateLimitShortCooldown(t *testing.T) {
|
||||
// 429 must NOT use the exponential backoff schedule: a rate-limited but
|
||||
// quota-rich source has to re-enter rotation after the short fixed window.
|
||||
p := newTestProvider(t, src("mock", "http://127.0.0.1:1", "openai", "m"))
|
||||
st := p.state("m")
|
||||
|
||||
// repeated 429s must stay at the fixed short cooldown, never escalate
|
||||
for i := 0; i < 15; i++ {
|
||||
p.ReportStatus("m", 429)
|
||||
}
|
||||
until := st.CooldownUntil()
|
||||
want := time.Now().Add(rateLimitCooldown).Unix()
|
||||
if until < want-2 || until > want+2 {
|
||||
t.Fatalf("429 cooldown = %d, want ~%d (fixed %v, not exponential)", until, want, rateLimitCooldown)
|
||||
}
|
||||
if st.FailCount() != 15 {
|
||||
t.Fatalf("fail count = %d, want 15 (counted but not escalating)", st.FailCount())
|
||||
}
|
||||
if st.Pref() != -15*int64(prefFailStep) && st.Pref() > int64(prefMin) {
|
||||
t.Fatalf("pref = %d", st.Pref())
|
||||
}
|
||||
if p.ModelAvailable("m") {
|
||||
t.Fatal("model must be cooling right after a 429")
|
||||
}
|
||||
}
|
||||
|
||||
func TestThrottleSpacingAndCancel(t *testing.T) {
|
||||
p := newTestProvider(t, src("mock", "http://127.0.0.1:1", "openai", "m"))
|
||||
p.cfg.RPM = 120 // gap = 500ms
|
||||
p.rpmGap = time.Minute / time.Duration(p.cfg.RPM)
|
||||
|
||||
start := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := p.Throttle(context.Background()); err != nil {
|
||||
t.Fatalf("throttle %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
// first call passes immediately, the next two wait one gap each
|
||||
want := 2 * time.Minute / time.Duration(p.cfg.RPM)
|
||||
if elapsed < want-time.Duration(100*time.Millisecond) || elapsed > want+time.Second {
|
||||
t.Fatalf("3 throttled calls took %v, want ~%v", elapsed, want)
|
||||
}
|
||||
|
||||
// unlimited source: Throttle is a no-op
|
||||
p2 := newTestProvider(t, src("free", "http://127.0.0.1:1", "openai", "m"))
|
||||
if err := p2.Throttle(context.Background()); err != nil {
|
||||
t.Fatalf("unlimited throttle: %v", err)
|
||||
}
|
||||
|
||||
// cancelled context aborts a pending window reservation
|
||||
p3 := newTestProvider(t, src("slow", "http://127.0.0.1:1", "openai", "m"))
|
||||
p3.cfg.RPM = 6 // 10s gap
|
||||
p3.rpmGap = time.Minute / time.Duration(p3.cfg.RPM)
|
||||
// first call passes immediately (fresh provider) and reserves the next window
|
||||
if err := p3.Throttle(context.Background()); err != nil {
|
||||
t.Fatalf("first throttle: %v", err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
||||
defer cancel()
|
||||
// second call must wait ~10s for the reserved window; the short deadline aborts it
|
||||
start = time.Now()
|
||||
err := p3.Throttle(ctx)
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("cancelled throttle = %v, want DeadlineExceeded", err)
|
||||
}
|
||||
if time.Since(start) > time.Second {
|
||||
t.Fatalf("cancel took %v, want fast abort", time.Since(start))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTryAcquire(t *testing.T) {
|
||||
p := newTestProvider(t, src("mock", "http://127.0.0.1:1", "openai", "m"))
|
||||
p.cfg.MaxConcurrent = 1
|
||||
|
||||
Reference in New Issue
Block a user