feat: AUTO chain rewrite — silent failover+busy skip+pref round-robin+503 tier summary; chain edits reset slot cooldowns (P0/P1); stats by_status + audit jsonl rotation; UI priority-page health badges & status-code card; ctx-menu capture-phase close (outside-press guard); main.go ops warnings; local bundled-Lua verified tests (3 latent bugs fixed); plan.md

internal/provider/provider.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"llmsproxy/internal/config"
@@ -20,36 +21,109 @@ import (
 	"llmsproxy/internal/types"
 )
 
-// health tracks availability with exponential backoff.
-type health struct {
-	failCount        int
-	unavailableUntil time.Time
-	permanent        bool
+// ---- per-(source,model) scheduling state ----
+
+// ModelState is the scheduling state of one (source, model) pair: a soft
+// preference score used to order candidates within a priority tier, a
+// consecutive-failure counter driving exponential cooldown, and a hard
+// cooldown deadline. All fields are atomic and cooldown expiry is evaluated
+// lazily (no timers, no goroutines). A model is never permanently blacklisted:
+// cooldown always expires and any success resets the state, so a fixed
+// upstream recovers on its own.
+type ModelState struct {
+	pref          atomic.Int64 // +1 per success / -5 per failure, clamped
+	failCount     atomic.Int64
+	cooldownUntil atomic.Int64 // unix seconds; 0 = schedulable
 }
 
-func (h *health) reset() { h.failCount = 0; h.unavailableUntil = time.Time{}; h.permanent = false }
+const (
+	prefFailStep = 5
+	prefMin      = -20
+	prefMax      = 20
+	backoffBase  = 5 * time.Second
+	backoffCap   = 30 * time.Minute
+	// failures at/above this count back off at the capped duration
+	backoffCapN = 10
+)
 
-func (h *health) available() bool {
-	if h.permanent {
-		return false
+func clampPref(a *atomic.Int64, lo, hi int64) {
+	for {
+		cur := a.Load()
+		if cur < lo {
+			if a.CompareAndSwap(cur, lo) {
+				return
+			}
+			continue
+		}
+		if cur > hi {
+			if a.CompareAndSwap(cur, hi) {
+				return
+			}
+			continue
+		}
+		return
 	}
-	return time.Now().After(h.unavailableUntil)
 }
 
-func (h *health) backoff() {
-	h.failCount++
-	cooldown := 5 * time.Second * time.Duration(1<<(h.failCount-1))
-	if cooldown > 30*time.Minute {
-		cooldown = 30 * time.Minute
+// Available reports whether the model may be scheduled right now (cooldown
+// expired or not yet set).
+func (s *ModelState) Available() bool {
+	return s.cooldownUntil.Load() <= time.Now().Unix()
+}
+
+// RecordFailure counts one consecutive failure and schedules exponential
+// cooldown (5s, 10s, 20s … capped at 30min). auth marks 401/403 credential
+// failures: it jumps straight to the capped cooldown and doubles the
+// preference penalty, but the model still recovers when the cooldown expires.
+func (s *ModelState) RecordFailure(auth bool) {
+	n := s.failCount.Add(1)
+	if auth && n < backoffCapN {
+		n = backoffCapN
+		s.failCount.Store(n) // persist the cap so FailCount() reports it too
 	}
-	h.unavailableUntil = time.Now().Add(cooldown)
+	var cd time.Duration
+	if n >= backoffCapN {
+		cd = backoffCap
+	} else {
+		cd = backoffBase * time.Duration(1<<(n-1))
+		if cd > backoffCap {
+			cd = backoffCap
+		}
+	}
+	s.cooldownUntil.Store(time.Now().Add(cd).Unix())
+	pen := int64(prefFailStep)
+	if auth {
+		pen *= 2
+	}
+	s.pref.Add(-pen)
+	clampPref(&s.pref, prefMin, prefMax)
 }
 
-func (h *health) markPermanent() {
-	h.permanent = true
-	h.unavailableUntil = time.Time{}
+// RecordSuccess resets the failure counter and cooldown and bumps the
+// preference score by one.
+func (s *ModelState) RecordSuccess() {
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+	s.pref.Add(1)
+	clampPref(&s.pref, prefMin, prefMax)
 }
 
+func (s *ModelState) reset() {
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+	s.pref.Store(0)
+}
+
+// Pref is the current preference score (higher = preferred).
+func (s *ModelState) Pref() int64 { return s.pref.Load() }
+
+// FailCount is the number of consecutive failures.
+func (s *ModelState) FailCount() int64 { return s.failCount.Load() }
+
+// CooldownUntil is the unix timestamp until which the model is cooled; 0 when
+// schedulable.
+func (s *ModelState) CooldownUntil() int64 { return s.cooldownUntil.Load() }
+
 // Provider is a single configured upstream LLM source.
 type Provider struct {
 	cfg     config.Source
@@ -59,11 +133,11 @@ type Provider struct {
 
 	mu        sync.Mutex
 	sem       chan struct{}
-	health    health
+	states    map[string]*ModelState // key = model id
 	lastProbe struct {
-		ok   bool
-		err  string
-		at   int64
+		ok  bool
+		err string
+		at  int64
 	}
 }
 
@@ -74,17 +148,21 @@ func New(cfg config.Source, vm *lua.VM) *Provider {
 		adapter: cfg.Adapter,
 		client:  &http.Client{Timeout: cfg.Timeout},
 		sem:     make(chan struct{}, cfg.MaxConcurrent),
+		states:  map[string]*ModelState{},
 	}
 	if cfg.MaxConcurrent <= 0 {
 		p.sem = nil
 	}
+	for _, m := range cfg.Models {
+		p.states[m.ID] = &ModelState{}
+	}
 	return p
 }
 
-func (p *Provider) Name() string            { return p.cfg.Name }
-func (p *Provider) Adapter() string         { return p.cfg.Adapter }
-func (p *Provider) MaxConcurrent() int      { return p.cfg.MaxConcurrent }
-func (p *Provider) Config() *config.Source  { return &p.cfg }
+func (p *Provider) Name() string           { return p.cfg.Name }
+func (p *Provider) Adapter() string        { return p.cfg.Adapter }
+func (p *Provider) MaxConcurrent() int     { return p.cfg.MaxConcurrent }
+func (p *Provider) Config() *config.Source { return &p.cfg }
 
 // Models returns the model ids exposed by this source.
 func (p *Provider) Models() []string {
@@ -174,10 +252,59 @@ func (p *Provider) ImageURL() string {
 
 // ---- availability ----
 
+// ErrBusy is returned when every concurrency slot of a source is in use. It
+// is a soft signal: schedulers skip a busy candidate without recording any
+// failure (busy is not a failure) and gateways map it to HTTP 429. It aliases
+// types.ErrBusy so the scheduler layer (which must not depend on this package)
+// can detect busy via the shared sentinel.
+var ErrBusy = types.ErrBusy
+
+// Available reports whether the source is schedulable at source level: at
+// least one of its models is not cooling down. Per-model scheduling decisions
+// must use ModelAvailable instead.
 func (p *Provider) Available() bool {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	return p.health.available()
+	for _, s := range p.states {
+		if s.Available() {
+			return true
+		}
+	}
+	return false
+}
+
+// ModelAvailable reports whether the exact model is schedulable right now
+// (its cooldown expired). An unknown model id is treated as available.
+func (p *Provider) ModelAvailable(model string) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if s, ok := p.states[model]; ok {
+		return s.Available()
+	}
+	return true
+}
+
+// Pref returns the adaptive preference score of a model (higher = preferred
+// within a priority tier). Used by the AUTO chain to order same-tier slots.
+func (p *Provider) Pref(model string) int64 {
+	return p.state(model).Pref()
+}
+
+// ResetModelCooldown clears the cooldown and failure counter of a single
+// model while preserving its preference score. Called after AUTO-chain edits
+// so edited slots become schedulable immediately (a stored preference for a
+// reliably good model is kept).
+func (p *Provider) ResetModelCooldown(model string) {
+	s := p.state(model)
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+}
+
+// ModelHealthInfo exposes the per-model scheduling state for the web UI.
+// An unknown model id reports zeros.
+func (p *Provider) ModelHealthInfo(model string) (pref, failCount, cooldownUntil int64) {
+	s := p.state(model)
+	return s.Pref(), s.FailCount(), s.CooldownUntil()
 }
 
 // Probe performs a lightweight reachability + auth check against the source.
@@ -274,35 +401,106 @@ func (p *Provider) LastProbe() (bool, string, int64) {
 	return p.lastProbe.ok, p.lastProbe.err, p.lastProbe.at
 }
 
-// ReportStatus records an upstream HTTP status for backoff decisions.
-func (p *Provider) ReportStatus(code int) {
+// state returns the ModelState for a model id, creating it on first use so
+// dynamically requested models are still tracked. The registry keeps states
+// alive across provider rebuilds only for configured models; a lazily created
+// state simply lives for the provider's lifetime.
+func (p *Provider) state(model string) *ModelState {
 	p.mu.Lock()
 	defer p.mu.Unlock()
+	s, ok := p.states[model]
+	if !ok {
+		s = &ModelState{}
+		p.states[model] = s
+	}
+	return s
+}
+
+// RecordFailure records a failed downstream attempt on model: consecutive
+// failure count +1 and exponential cooldown (5s·2^n, capped at 30min).
+// code 401/403 is treated as a credential problem: the cooldown jumps to the
+// cap and the preference penalty doubles, but the model still recovers when
+// the cooldown expires (no permanent blacklist). code 0 = transport failure.
+func (p *Provider) RecordFailure(model string, code int) {
+	p.state(model).RecordFailure(code == 401 || code == 403)
+}
+
+// RecordSuccess resets the model's failure counter / cooldown and bumps its
+// preference by one.
+func (p *Provider) RecordSuccess(model string) {
+	p.state(model).RecordSuccess()
+}
+
+// ReportStatus records an upstream HTTP status for the given model and drives
+// the (source, model) backoff state. 401/403 → capped self-healing cooldown
+// with doubled penalty; 429 and 5xx → normal exponential backoff. Other codes
+// (400 client schema errors, 402 billing errors) are not penalized here —
+// they surface via the status page / audit instead.
+func (p *Provider) ReportStatus(model string, code int) {
 	if code == 401 || code == 403 {
-		p.health.markPermanent()
+		p.RecordFailure(model, code)
 		return
 	}
 	if code >= 500 || code == 429 {
-		p.health.backoff()
+		p.RecordFailure(model, code)
 	}
 }
 
-func (p *Provider) reportError() {
+// ResetHealth resets the scheduling state of every model of this source
+// (cooldown and preference to zero), so the source becomes fully schedulable
+// again. Called after AUTO-chain edits and from the admin UI.
+func (p *Provider) ResetHealth() {
 	p.mu.Lock()
-	p.health.backoff()
-	p.mu.Unlock()
+	defer p.mu.Unlock()
+	for _, s := range p.states {
+		s.reset()
+	}
 }
 
-func (p *Provider) reportOK() {
+// HealthInfo exposes the source-level backoff state for the status page: the
+// highest failure count and the latest cooldown deadline across all models of
+// this source. permanent is always false — the permanent-blacklist semantics
+// were removed; every cooldown expires on its own.
+func (p *Provider) HealthInfo() (failCount int, until time.Time, permanent bool) {
 	p.mu.Lock()
-	p.health.reset()
-	p.mu.Unlock()
+	defer p.mu.Unlock()
+	now := time.Now().Unix()
+	for _, s := range p.states {
+		if n := int(s.FailCount()); n > failCount {
+			failCount = n
+		}
+		if t := s.CooldownUntil(); t > now && t > until.Unix() {
+			until = time.Unix(t, 0)
+		}
+	}
+	return failCount, until, false
 }
 
 // ---- concurrency limiting ----
 
+// TryAcquire takes one concurrency slot without blocking: it returns nil when
+// a slot is free and ErrBusy when the source is at capacity. A nil semaphore
+// (MaxConcurrent <= 0) means unlimited and always succeeds. TryAcquire is the
+// single busy/idle signal for schedulers; a busy source is skipped, never
+// penalized.
+func (p *Provider) TryAcquire(ctx context.Context) error {
+	if p.sem == nil {
+		return nil
+	}
+	select {
+	case p.sem <- struct{}{}:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+		return ErrBusy
+	}
+}
+
 // Acquire waits for a free concurrency slot (bounded by cfg.QueueTimeout),
-// or context cancel. The HTTP call itself is not truncated.
+// or context cancel. The HTTP call itself is not truncated. Direct requests
+// historically queued here; the scheduler now prefers TryAcquire so a full
+// source fails fast instead of blocking the whole chain.
 func (p *Provider) Acquire(ctx context.Context) error {
 	if p.sem == nil {
 		return nil
@@ -367,11 +565,14 @@ func (p *Provider) buildHeaders(body, url string) (http.Header, error) {
 // ---- chat ----
 
 // Chat performs a non-streaming round trip and returns the unified response.
+// It fails fast with ErrBusy when the source is at capacity; success/failure
+// is recorded against the resolved (source, model) scheduling state.
 func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.UnifiedResponse, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
 	defer p.Release()
+	model := p.ModelFor(req.Model)
 
 	body, err := marshalTransform(p.vm, p.adapter, "transform_request", req)
 	if err != nil {
@@ -383,11 +584,11 @@ func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.Uni
 	}
 	raw, status, err := p.do(ctx, p.URL(), body, hdrs)
 	if err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		return nil, err
 	}
 	if status != 200 {
-		p.ReportStatus(status)
+		p.ReportStatus(model, status)
 		return nil, fmt.Errorf("api error %d: %s", status, truncate(raw, 500))
 	}
 	unified, err := p.vm.Transform(p.adapter, "transform_response", raw)
@@ -398,15 +599,21 @@ func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.Uni
 	if err := json.Unmarshal([]byte(unified), &out); err != nil {
 		return nil, fmt.Errorf("unmarshal unified response: %w (body: %s)", err, unified)
 	}
-	p.reportOK()
+	p.RecordSuccess(model)
 	return &out, nil
 }
 
-// ChatStream performs a streaming round trip, emitting unified chunks.
+// ChatStream performs a streaming round trip, emitting unified chunks. It
+// fails fast with ErrBusy when the source is at capacity. Only a failure
+// before the first chunk (connect error or non-200 status) is recorded
+// against the (source, model) state; afterwards the stream is pinned. A clean
+// end ([DONE] or EOF without read errors, and no client disconnect) counts as
+// success and resets the cooldown.
 func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-chan types.UnifiedChunk, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
+	model := p.ModelFor(req.Model)
 	req.Stream = true
 	body, err := marshalTransform(p.vm, p.adapter, "transform_request", req)
 	if err != nil {
@@ -432,14 +639,14 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
 	ch := make(chan types.UnifiedChunk, 64)
 	sel := <-rc
 	if sel.err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		p.Release()
 		return nil, sel.err
 	}
 	if sel.resp.StatusCode != 200 {
 		raw, _ := io.ReadAll(sel.resp.Body)
 		sel.resp.Body.Close()
-		p.ReportStatus(sel.resp.StatusCode)
+		p.ReportStatus(model, sel.resp.StatusCode)
 		p.Release()
 		return nil, fmt.Errorf("api error %d: %s", sel.resp.StatusCode, truncate(string(raw), 500))
 	}
@@ -485,16 +692,25 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
 				return
 			}
 		}
+		// The stream ended cleanly ([DONE] seen or EOF without an upstream
+		// read error): record success so a previously cooled model can be
+		// retried. A client disconnect or mid-stream read error is neither
+		// success nor failure for scheduling purposes.
+		if ctx.Err() == nil && scanner.Err() == nil {
+			p.RecordSuccess(model)
+		}
 	}()
 	return ch, nil
 }
 
-// Image generates images via /v1/images/generations.
+// Image generates images via /v1/images/generations. Same scheduling-state
+// accounting as Chat: fail fast on busy, record per (source, model).
 func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*types.UnifiedResponse, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
 	defer p.Release()
+	model := p.ModelFor(req.Model)
 
 	b, _ := json.Marshal(req)
 	transformed, err := p.vm.Transform(p.adapter+"_image", "transform_request", string(b))
@@ -508,11 +724,11 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 	}
 	raw, status, err := p.do(ctx, p.ImageURL(), transformed, hdrs)
 	if err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		return nil, err
 	}
 	if status != 200 {
-		p.ReportStatus(status)
+		p.ReportStatus(model, status)
 		return nil, fmt.Errorf("image api error %d: %s", status, truncate(raw, 500))
 	}
 	var out types.UnifiedResponse
@@ -520,7 +736,7 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 	unified, terr := p.vm.Transform(p.adapter+"_image", "transform_response", raw)
 	if terr == nil && unified != raw {
 		if err := json.Unmarshal([]byte(unified), &out); err == nil {
-			p.reportOK()
+			p.RecordSuccess(model)
 			return &out, nil
 		}
 	}
@@ -529,7 +745,7 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 		return nil, fmt.Errorf("unmarshal image response: %w", err)
 	}
 	out.ImageData = img.Data
-	p.reportOK()
+	p.RecordSuccess(model)
 	return &out, nil
 }
 
@@ -593,4 +809,4 @@ func truncate(s string, n int) string {
 		return s
 	}
 	return s[:n] + "..."
-}
+}

internal/provider/provider_test.go

@@ -3,11 +3,12 @@ package provider
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"path/filepath"
-	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -90,6 +91,49 @@ func TestProviderChatStream(t *testing.T) {
 	}
 }
 
+// TestStreamSuccessClearsBackoff guards P6: a clean streaming end must reset
+// a previously cooled (source, model) pair.
+func TestStreamSuccessClearsBackoff(t *testing.T) {
+	var fail atomic.Bool
+	fail.Store(true)
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if fail.Load() {
+			w.WriteHeader(500)
+			return
+		}
+		w.Header().Set("Content-Type", "text/event-stream")
+		fmt.Fprint(w, "data: {\"choices\":[{\"delta\":{\"content\":\"a\"}}]}\n\n")
+		fmt.Fprint(w, "data: [DONE]\n\n")
+	}))
+	defer up.Close()
+	p := newTestProvider(t, src("mock", up.URL, "openai", "m"))
+	if _, err := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+	}); err == nil {
+		t.Fatal("expected first chat to fail")
+	}
+	if p.ModelAvailable("m") {
+		t.Fatal("m must be cooling after the failed chat")
+	}
+	fail.Store(false)
+	ch, err := p.ChatStream(context.Background(), &types.ChatRequest{
+		Model:    "m",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+	})
+	if err != nil {
+		t.Fatalf("stream: %v", err)
+	}
+	for range ch {
+	}
+	if !p.ModelAvailable("m") {
+		t.Fatal("clean stream must clear the cooldown")
+	}
+	if st := p.state("m"); st.FailCount() != 0 {
+		t.Fatalf("fail count after clean stream = %d", st.FailCount())
+	}
+}
+
 func TestProviderImage(t *testing.T) {
 	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, `{"created":123,"data":[{"b64_json":"QUJD"}]}`)
@@ -105,12 +149,71 @@ func TestProviderImage(t *testing.T) {
 	}
 }
 
-func TestProviderBackoff(t *testing.T) {
+func TestProviderBackoffPerModel(t *testing.T) {
 	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(500)
 		fmt.Fprint(w, "boom")
 	}))
 	defer up.Close()
+	// two models on one source: a failure on m1 must not blacklist m2
+	p := newTestProvider(t, src("mock", up.URL, "openai", "m1", "m2"))
+	_, err := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m1",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+	})
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	if p.ModelAvailable("m1") {
+		t.Fatal("expected m1 to be cooling down")
+	}
+	if !p.ModelAvailable("m2") {
+		t.Fatal("m2 must stay schedulable (per-model isolation)")
+	}
+	if !p.Available() {
+		t.Fatal("source must stay available while any model is schedulable")
+	}
+	if st := p.state("m1"); st.FailCount() != 1 {
+		t.Fatalf("fail count = %d, want 1", st.FailCount())
+	}
+}
+
+func TestProviderAuthFailureSelfHeals(t *testing.T) {
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(401)
+		fmt.Fprint(w, `{"error":"API_KEY_DISABLED"}`)
+	}))
+	defer up.Close()
+	p := newTestProvider(t, src("mock2", up.URL, "openai", "m2"))
+	_, err := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m2",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+	})
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	st := p.state("m2")
+	if st.FailCount() != backoffCapN {
+		t.Fatalf("auth failure must jump to capped count, got %d", st.FailCount())
+	}
+	if until := st.CooldownUntil(); until <= time.Now().Add(25*time.Minute).Unix() {
+		t.Fatalf("auth failure must cool near the cap (until=%d)", until)
+	}
+	if st.Pref() != -2*int64(prefFailStep) {
+		t.Fatalf("auth failure pref penalty must be doubled, got %d", st.Pref())
+	}
+	// not permanent: the reset channel and a later success both restore it
+	st.reset()
+	if !p.ModelAvailable("m2") {
+		t.Fatal("reset must restore schedulability")
+	}
+}
+
+func TestModelStateCooldownAndRecovery(t *testing.T) {
+	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(500)
+	}))
+	defer up.Close()
 	p := newTestProvider(t, src("mock", up.URL, "openai", "m"))
 	_, err := p.Chat(context.Background(), &types.ChatRequest{
 		Model:    "m",
@@ -119,55 +222,79 @@ func TestProviderBackoff(t *testing.T) {
 	if err == nil {
 		t.Fatal("expected error")
 	}
-	if p.Available() {
-		t.Fatal("expected provider to be in backoff")
+	st := p.state("m")
+	if st.FailCount() != 1 {
+		t.Fatalf("fail count = %d", st.FailCount())
 	}
-	// 401 -> permanent
-	up2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(401)
-	}))
-	defer up2.Close()
-	p2 := newTestProvider(t, src("mock2", up2.URL, "openai", "m2"))
-	p2.Chat(context.Background(), &types.ChatRequest{Model: "m2", Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}}})
-	if p2.Available() {
-		t.Fatal("expected permanent unavailability on 401")
+	// one failure -> 5s cooldown from now
+	until := st.CooldownUntil()
+	want := time.Now().Add(backoffBase).Unix()
+	if until < want-2 || until > want+2 {
+		t.Fatalf("cooldown = %d, want ~%d", until, want)
+	}
+	// success resets everything and bumps the preference
+	st.RecordSuccess()
+	if !p.ModelAvailable("m") {
+		t.Fatal("success must clear cooldown")
+	}
+	if st.FailCount() != 0 {
+		t.Fatalf("fail count after success = %d", st.FailCount())
+	}
+	if st.Pref() != 1-int64(prefFailStep) {
+		t.Fatalf("pref after one failure (-5) then success (+1) = %d, want %d", st.Pref(), 1-int64(prefFailStep))
 	}
 }
 
-func TestProviderConcurrencyCap(t *testing.T) {
+func TestTryAcquire(t *testing.T) {
+	p := newTestProvider(t, src("mock", "http://127.0.0.1:1", "openai", "m"))
+	p.cfg.MaxConcurrent = 1
+	p.sem = make(chan struct{}, 1)
+	if err := p.TryAcquire(context.Background()); err != nil {
+		t.Fatalf("first acquire: %v", err)
+	}
+	if err := p.TryAcquire(context.Background()); !errors.Is(err, ErrBusy) {
+		t.Fatalf("second acquire = %v, want ErrBusy", err)
+	}
+	p.Release()
+	if err := p.TryAcquire(context.Background()); err != nil {
+		t.Fatalf("acquire after release: %v", err)
+	}
+	p.Release()
+}
+
+func TestChatBusyFailsFast(t *testing.T) {
 	release := make(chan struct{})
-	started := make(chan struct{}, 100)
+	started := make(chan struct{}, 10)
 	up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		started <- struct{}{}
 		<-release
 		fmt.Fprint(w, `{"choices":[{"message":{"content":"ok"}}]}`)
 	}))
 	defer up.Close()
-	// cap 2
 	p := newTestProvider(t, src("mock", up.URL, "openai", "m"))
-	p.cfg.MaxConcurrent = 2
-	p.sem = make(chan struct{}, 2)
+	p.cfg.MaxConcurrent = 1
+	p.sem = make(chan struct{}, 1)
 
-	var wg sync.WaitGroup
-	for i := 0; i < 6; i++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			p.Chat(context.Background(), &types.ChatRequest{Model: "m", Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}}})
-		}()
-	}
-	// wait until 2 requests started
-	deadline := time.Now().Add(2 * time.Second)
-	for len(started) < 2 {
-		if time.Now().After(deadline) {
-			t.Fatal("timeout waiting for first two")
-		}
-		time.Sleep(5 * time.Millisecond)
-	}
-	time.Sleep(100 * time.Millisecond)
-	if len(started) > 2 {
-		t.Fatalf("more than 2 concurrent: %d", len(started))
+	done := make(chan error, 1)
+	go func() {
+		_, err := p.Chat(context.Background(), &types.ChatRequest{
+			Model:    "m",
+			Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+		})
+		done <- err
+	}()
+	<-started // first request holds the only slot
+
+	// second request must fail fast with ErrBusy instead of queueing
+	_, err2 := p.Chat(context.Background(), &types.ChatRequest{
+		Model:    "m",
+		Messages: []types.ChatMessage{{Role: "user", Content: types.StringContent("x")}},
+	})
+	if !errors.Is(err2, ErrBusy) {
+		t.Fatalf("second chat err = %v, want ErrBusy", err2)
 	}
 	close(release)
-	wg.Wait()
-}
+	if err := <-done; err != nil {
+		t.Fatalf("first chat: %v", err)
+	}
+}

internal/provider/registry.go

@@ -75,63 +75,39 @@ func (r *Registry) ModelList() []string {
 	return out
 }
 
-// Resolve returns the ordered candidate providers to try for a request,
-// honoring explicit model selection or AUTO (priority order, healthy first).
+// Resolve returns the provider (or providers) serving a requested model,
+// owning no AUTO scheduling logic anymore: AUTO chat scheduling is driven by
+// the scheduler chain built from the runtime rules (see core/SaveAutoRules
+// and scheduler.Chain).
 //
-// model "" or "AUTO" -> all sources sorted by (priority desc, healthy first).
-// Otherwise the owning provider, if healthy; else its source anyway.
+// model "" or "AUTO" -> every provider in configured order. Used only by the
+// image path (which then filters to image-capable sources) and tool-call
+// anchoring; chat AUTO requests go through the chain instead.
+// Otherwise the owning provider; "source-model"/"source:model"/"source/model"
+// pinning resolves first; an unknown model resolves to nil (gateway answers
+// 404) instead of silently falling back to the AUTO chain.
 func (r *Registry) Resolve(model string) []*Provider {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 
 	model = strings.TrimSpace(model)
 	if model == "" || strings.EqualFold(model, "AUTO") {
-		// priority chain across all models
-		type cand struct {
-			prov     *Provider
-			priority int
-		}
-		var cands []cand
-		seen := map[string]bool{}
-		for _, p := range r.providers {
-			prio := -1
-			for _, m := range p.cfg.Models {
-				if m.Priority > prio {
-					prio = m.Priority
-				}
-			}
-			if prio < 0 {
-				prio = 0
-			}
-			cands = append(cands, cand{p, prio})
-			seen[p.Name()] = true
-		}
-		sort.SliceStable(cands, func(i, j int) bool {
-			if cands[i].priority != cands[j].priority {
-				return cands[i].priority > cands[j].priority
-			}
-			// healthy preferred at same priority
-			return cands[i].prov.Available() && !cands[j].prov.Available()
-		})
-		out := make([]*Provider, 0, len(cands))
-		for _, c := range cands {
-			out = append(out, c.prov)
-		}
+		out := make([]*Provider, len(r.providers))
+		copy(out, r.providers)
 		return out
 	}
 
-	// explicit model
-	if p, ok := r.byModel[strings.ToLower(model)]; ok {
-		// switch to the owning source but pin the model via request
-		return []*Provider{p}
-	}
 	// "source-model" / "source:model" / "source/model" pinning — disambiguates
 	// duplicate model ids across sources.
 	if p := r.ResolvePinned(model); p != nil {
 		return []*Provider{p}
 	}
-	// unknown model -> fall back to default/AUTO chain
-	return r.AUTOChain()
+	// explicit model
+	if p, ok := r.byModel[strings.ToLower(model)]; ok {
+		// switch to the owning source but pin the model via request
+		return []*Provider{p}
+	}
+	return nil
 }
 
 // EffectiveModel strips a "source-model" / "source:model" / "source/model"
@@ -174,11 +150,6 @@ func (r *Registry) ResolvePinned(model string) *Provider {
 	return nil
 }
 
-// AUTOChain returns the priority-sorted providers for AUTO.
-func (r *Registry) AUTOChain() []*Provider {
-	return r.Resolve("AUTO")
-}
-
 // ProviderForModel returns the provider owning the model id (nil if unknown).
 func (r *Registry) ProviderForModel(model string) *Provider {
 	r.mu.RLock()
@@ -220,15 +191,6 @@ func (r *Registry) ProviderForSlot(model, source string) *Provider {
 	return nil
 }
 
-// Default returns the highest-priority available provider.
-func (r *Registry) Default() *Provider {
-	chain := r.AUTOChain()
-	if len(chain) == 0 {
-		return nil
-	}
-	return chain[0]
-}
-
 // ModelStatus is a web-UI friendly snapshot per source.
 type SourceStatus struct {
 	Name          string   `json:"name"`
@@ -241,6 +203,9 @@ type SourceStatus struct {
 	LiveAvailable bool     `json:"live_available"`
 	LastError     string   `json:"last_error,omitempty"`
 	LastChecked   int64    `json:"last_checked,omitempty"`
+	FailCount     int      `json:"fail_count,omitempty"`
+	BackoffUntil  int64    `json:"backoff_until,omitempty"`
+	Permanent     bool     `json:"permanent,omitempty"`
 }
 
 // ProbeAll runs a live reachability check for every provider (in parallel).
@@ -267,19 +232,27 @@ func (r *Registry) Status() []SourceStatus {
 	out := make([]SourceStatus, 0, len(r.providers))
 	for _, p := range r.providers {
 		live, lastErr, lastAt := p.LastProbe()
-	s := SourceStatus{
-		Name:          p.Name(),
-		Adapter:       p.Adapter(),
-		BaseURL:       p.Config().BaseURL,
-		Models:        p.Models(),
-		Available:     p.Available(),
-		Healthy:       p.Available(),
-		MaxConcurrent: p.MaxConcurrent(),
-		LiveAvailable: live,
-		LastError:     lastErr,
-		LastChecked:   lastAt,
-	}
-	out = append(out, s)
+		fails, until, perm := p.HealthInfo()
+		backoffUntil := int64(0)
+		if !until.IsZero() {
+			backoffUntil = until.Unix()
+		}
+		s := SourceStatus{
+			Name:          p.Name(),
+			Adapter:       p.Adapter(),
+			BaseURL:       p.Config().BaseURL,
+			Models:        p.Models(),
+			Available:     p.Available(),
+			Healthy:       p.Available(),
+			MaxConcurrent: p.MaxConcurrent(),
+			LiveAvailable: live,
+			LastError:     lastErr,
+			LastChecked:   lastAt,
+			FailCount:     fails,
+			BackoffUntil:  backoffUntil,
+			Permanent:     perm,
+		}
+		out = append(out, s)
 	}
 	return out
 }