feat: AUTO chain rewrite — silent failover+busy skip+pref round-robin+503 tier summary; chain edits reset slot cooldowns (P0/P1); stats by_status + audit jsonl rotation; UI priority-page health badges & status-code card; ctx-menu capture-phase close (outside-press guard); main.go ops warnings; local bundled-Lua verified tests (3 latent bugs fixed); plan.md

internal/provider/provider.go

@@ -13,6 +13,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"llmsproxy/internal/config"
@@ -20,36 +21,109 @@ import (
 	"llmsproxy/internal/types"
 )
 
-// health tracks availability with exponential backoff.
-type health struct {
-	failCount        int
-	unavailableUntil time.Time
-	permanent        bool
+// ---- per-(source,model) scheduling state ----
+
+// ModelState is the scheduling state of one (source, model) pair: a soft
+// preference score used to order candidates within a priority tier, a
+// consecutive-failure counter driving exponential cooldown, and a hard
+// cooldown deadline. All fields are atomic and cooldown expiry is evaluated
+// lazily (no timers, no goroutines). A model is never permanently blacklisted:
+// cooldown always expires and any success resets the state, so a fixed
+// upstream recovers on its own.
+type ModelState struct {
+	pref          atomic.Int64 // +1 per success / -5 per failure, clamped
+	failCount     atomic.Int64
+	cooldownUntil atomic.Int64 // unix seconds; 0 = schedulable
 }
 
-func (h *health) reset() { h.failCount = 0; h.unavailableUntil = time.Time{}; h.permanent = false }
+const (
+	prefFailStep = 5
+	prefMin      = -20
+	prefMax      = 20
+	backoffBase  = 5 * time.Second
+	backoffCap   = 30 * time.Minute
+	// failures at/above this count back off at the capped duration
+	backoffCapN = 10
+)
 
-func (h *health) available() bool {
-	if h.permanent {
-		return false
+func clampPref(a *atomic.Int64, lo, hi int64) {
+	for {
+		cur := a.Load()
+		if cur < lo {
+			if a.CompareAndSwap(cur, lo) {
+				return
+			}
+			continue
+		}
+		if cur > hi {
+			if a.CompareAndSwap(cur, hi) {
+				return
+			}
+			continue
+		}
+		return
 	}
-	return time.Now().After(h.unavailableUntil)
 }
 
-func (h *health) backoff() {
-	h.failCount++
-	cooldown := 5 * time.Second * time.Duration(1<<(h.failCount-1))
-	if cooldown > 30*time.Minute {
-		cooldown = 30 * time.Minute
+// Available reports whether the model may be scheduled right now (cooldown
+// expired or not yet set).
+func (s *ModelState) Available() bool {
+	return s.cooldownUntil.Load() <= time.Now().Unix()
+}
+
+// RecordFailure counts one consecutive failure and schedules exponential
+// cooldown (5s, 10s, 20s … capped at 30min). auth marks 401/403 credential
+// failures: it jumps straight to the capped cooldown and doubles the
+// preference penalty, but the model still recovers when the cooldown expires.
+func (s *ModelState) RecordFailure(auth bool) {
+	n := s.failCount.Add(1)
+	if auth && n < backoffCapN {
+		n = backoffCapN
+		s.failCount.Store(n) // persist the cap so FailCount() reports it too
 	}
-	h.unavailableUntil = time.Now().Add(cooldown)
+	var cd time.Duration
+	if n >= backoffCapN {
+		cd = backoffCap
+	} else {
+		cd = backoffBase * time.Duration(1<<(n-1))
+		if cd > backoffCap {
+			cd = backoffCap
+		}
+	}
+	s.cooldownUntil.Store(time.Now().Add(cd).Unix())
+	pen := int64(prefFailStep)
+	if auth {
+		pen *= 2
+	}
+	s.pref.Add(-pen)
+	clampPref(&s.pref, prefMin, prefMax)
 }
 
-func (h *health) markPermanent() {
-	h.permanent = true
-	h.unavailableUntil = time.Time{}
+// RecordSuccess resets the failure counter and cooldown and bumps the
+// preference score by one.
+func (s *ModelState) RecordSuccess() {
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+	s.pref.Add(1)
+	clampPref(&s.pref, prefMin, prefMax)
 }
 
+func (s *ModelState) reset() {
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+	s.pref.Store(0)
+}
+
+// Pref is the current preference score (higher = preferred).
+func (s *ModelState) Pref() int64 { return s.pref.Load() }
+
+// FailCount is the number of consecutive failures.
+func (s *ModelState) FailCount() int64 { return s.failCount.Load() }
+
+// CooldownUntil is the unix timestamp until which the model is cooled; 0 when
+// schedulable.
+func (s *ModelState) CooldownUntil() int64 { return s.cooldownUntil.Load() }
+
 // Provider is a single configured upstream LLM source.
 type Provider struct {
 	cfg     config.Source
@@ -59,11 +133,11 @@ type Provider struct {
 
 	mu        sync.Mutex
 	sem       chan struct{}
-	health    health
+	states    map[string]*ModelState // key = model id
 	lastProbe struct {
-		ok   bool
-		err  string
-		at   int64
+		ok  bool
+		err string
+		at  int64
 	}
 }
 
@@ -74,17 +148,21 @@ func New(cfg config.Source, vm *lua.VM) *Provider {
 		adapter: cfg.Adapter,
 		client:  &http.Client{Timeout: cfg.Timeout},
 		sem:     make(chan struct{}, cfg.MaxConcurrent),
+		states:  map[string]*ModelState{},
 	}
 	if cfg.MaxConcurrent <= 0 {
 		p.sem = nil
 	}
+	for _, m := range cfg.Models {
+		p.states[m.ID] = &ModelState{}
+	}
 	return p
 }
 
-func (p *Provider) Name() string            { return p.cfg.Name }
-func (p *Provider) Adapter() string         { return p.cfg.Adapter }
-func (p *Provider) MaxConcurrent() int      { return p.cfg.MaxConcurrent }
-func (p *Provider) Config() *config.Source  { return &p.cfg }
+func (p *Provider) Name() string           { return p.cfg.Name }
+func (p *Provider) Adapter() string        { return p.cfg.Adapter }
+func (p *Provider) MaxConcurrent() int     { return p.cfg.MaxConcurrent }
+func (p *Provider) Config() *config.Source { return &p.cfg }
 
 // Models returns the model ids exposed by this source.
 func (p *Provider) Models() []string {
@@ -174,10 +252,59 @@ func (p *Provider) ImageURL() string {
 
 // ---- availability ----
 
+// ErrBusy is returned when every concurrency slot of a source is in use. It
+// is a soft signal: schedulers skip a busy candidate without recording any
+// failure (busy is not a failure) and gateways map it to HTTP 429. It aliases
+// types.ErrBusy so the scheduler layer (which must not depend on this package)
+// can detect busy via the shared sentinel.
+var ErrBusy = types.ErrBusy
+
+// Available reports whether the source is schedulable at source level: at
+// least one of its models is not cooling down. Per-model scheduling decisions
+// must use ModelAvailable instead.
 func (p *Provider) Available() bool {
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	return p.health.available()
+	for _, s := range p.states {
+		if s.Available() {
+			return true
+		}
+	}
+	return false
+}
+
+// ModelAvailable reports whether the exact model is schedulable right now
+// (its cooldown expired). An unknown model id is treated as available.
+func (p *Provider) ModelAvailable(model string) bool {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if s, ok := p.states[model]; ok {
+		return s.Available()
+	}
+	return true
+}
+
+// Pref returns the adaptive preference score of a model (higher = preferred
+// within a priority tier). Used by the AUTO chain to order same-tier slots.
+func (p *Provider) Pref(model string) int64 {
+	return p.state(model).Pref()
+}
+
+// ResetModelCooldown clears the cooldown and failure counter of a single
+// model while preserving its preference score. Called after AUTO-chain edits
+// so edited slots become schedulable immediately (a stored preference for a
+// reliably good model is kept).
+func (p *Provider) ResetModelCooldown(model string) {
+	s := p.state(model)
+	s.failCount.Store(0)
+	s.cooldownUntil.Store(0)
+}
+
+// ModelHealthInfo exposes the per-model scheduling state for the web UI.
+// An unknown model id reports zeros.
+func (p *Provider) ModelHealthInfo(model string) (pref, failCount, cooldownUntil int64) {
+	s := p.state(model)
+	return s.Pref(), s.FailCount(), s.CooldownUntil()
 }
 
 // Probe performs a lightweight reachability + auth check against the source.
@@ -274,35 +401,106 @@ func (p *Provider) LastProbe() (bool, string, int64) {
 	return p.lastProbe.ok, p.lastProbe.err, p.lastProbe.at
 }
 
-// ReportStatus records an upstream HTTP status for backoff decisions.
-func (p *Provider) ReportStatus(code int) {
+// state returns the ModelState for a model id, creating it on first use so
+// dynamically requested models are still tracked. The registry keeps states
+// alive across provider rebuilds only for configured models; a lazily created
+// state simply lives for the provider's lifetime.
+func (p *Provider) state(model string) *ModelState {
 	p.mu.Lock()
 	defer p.mu.Unlock()
+	s, ok := p.states[model]
+	if !ok {
+		s = &ModelState{}
+		p.states[model] = s
+	}
+	return s
+}
+
+// RecordFailure records a failed downstream attempt on model: consecutive
+// failure count +1 and exponential cooldown (5s·2^n, capped at 30min).
+// code 401/403 is treated as a credential problem: the cooldown jumps to the
+// cap and the preference penalty doubles, but the model still recovers when
+// the cooldown expires (no permanent blacklist). code 0 = transport failure.
+func (p *Provider) RecordFailure(model string, code int) {
+	p.state(model).RecordFailure(code == 401 || code == 403)
+}
+
+// RecordSuccess resets the model's failure counter / cooldown and bumps its
+// preference by one.
+func (p *Provider) RecordSuccess(model string) {
+	p.state(model).RecordSuccess()
+}
+
+// ReportStatus records an upstream HTTP status for the given model and drives
+// the (source, model) backoff state. 401/403 → capped self-healing cooldown
+// with doubled penalty; 429 and 5xx → normal exponential backoff. Other codes
+// (400 client schema errors, 402 billing errors) are not penalized here —
+// they surface via the status page / audit instead.
+func (p *Provider) ReportStatus(model string, code int) {
 	if code == 401 || code == 403 {
-		p.health.markPermanent()
+		p.RecordFailure(model, code)
 		return
 	}
 	if code >= 500 || code == 429 {
-		p.health.backoff()
+		p.RecordFailure(model, code)
 	}
 }
 
-func (p *Provider) reportError() {
+// ResetHealth resets the scheduling state of every model of this source
+// (cooldown and preference to zero), so the source becomes fully schedulable
+// again. Called after AUTO-chain edits and from the admin UI.
+func (p *Provider) ResetHealth() {
 	p.mu.Lock()
-	p.health.backoff()
-	p.mu.Unlock()
+	defer p.mu.Unlock()
+	for _, s := range p.states {
+		s.reset()
+	}
 }
 
-func (p *Provider) reportOK() {
+// HealthInfo exposes the source-level backoff state for the status page: the
+// highest failure count and the latest cooldown deadline across all models of
+// this source. permanent is always false — the permanent-blacklist semantics
+// were removed; every cooldown expires on its own.
+func (p *Provider) HealthInfo() (failCount int, until time.Time, permanent bool) {
 	p.mu.Lock()
-	p.health.reset()
-	p.mu.Unlock()
+	defer p.mu.Unlock()
+	now := time.Now().Unix()
+	for _, s := range p.states {
+		if n := int(s.FailCount()); n > failCount {
+			failCount = n
+		}
+		if t := s.CooldownUntil(); t > now && t > until.Unix() {
+			until = time.Unix(t, 0)
+		}
+	}
+	return failCount, until, false
 }
 
 // ---- concurrency limiting ----
 
+// TryAcquire takes one concurrency slot without blocking: it returns nil when
+// a slot is free and ErrBusy when the source is at capacity. A nil semaphore
+// (MaxConcurrent <= 0) means unlimited and always succeeds. TryAcquire is the
+// single busy/idle signal for schedulers; a busy source is skipped, never
+// penalized.
+func (p *Provider) TryAcquire(ctx context.Context) error {
+	if p.sem == nil {
+		return nil
+	}
+	select {
+	case p.sem <- struct{}{}:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	default:
+		return ErrBusy
+	}
+}
+
 // Acquire waits for a free concurrency slot (bounded by cfg.QueueTimeout),
-// or context cancel. The HTTP call itself is not truncated.
+// or context cancel. The HTTP call itself is not truncated. Direct requests
+// historically queued here; the scheduler now prefers TryAcquire so a full
+// source fails fast instead of blocking the whole chain.
 func (p *Provider) Acquire(ctx context.Context) error {
 	if p.sem == nil {
 		return nil
@@ -367,11 +565,14 @@ func (p *Provider) buildHeaders(body, url string) (http.Header, error) {
 // ---- chat ----
 
 // Chat performs a non-streaming round trip and returns the unified response.
+// It fails fast with ErrBusy when the source is at capacity; success/failure
+// is recorded against the resolved (source, model) scheduling state.
 func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.UnifiedResponse, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
 	defer p.Release()
+	model := p.ModelFor(req.Model)
 
 	body, err := marshalTransform(p.vm, p.adapter, "transform_request", req)
 	if err != nil {
@@ -383,11 +584,11 @@ func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.Uni
 	}
 	raw, status, err := p.do(ctx, p.URL(), body, hdrs)
 	if err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		return nil, err
 	}
 	if status != 200 {
-		p.ReportStatus(status)
+		p.ReportStatus(model, status)
 		return nil, fmt.Errorf("api error %d: %s", status, truncate(raw, 500))
 	}
 	unified, err := p.vm.Transform(p.adapter, "transform_response", raw)
@@ -398,15 +599,21 @@ func (p *Provider) Chat(ctx context.Context, req *types.ChatRequest) (*types.Uni
 	if err := json.Unmarshal([]byte(unified), &out); err != nil {
 		return nil, fmt.Errorf("unmarshal unified response: %w (body: %s)", err, unified)
 	}
-	p.reportOK()
+	p.RecordSuccess(model)
 	return &out, nil
 }
 
-// ChatStream performs a streaming round trip, emitting unified chunks.
+// ChatStream performs a streaming round trip, emitting unified chunks. It
+// fails fast with ErrBusy when the source is at capacity. Only a failure
+// before the first chunk (connect error or non-200 status) is recorded
+// against the (source, model) state; afterwards the stream is pinned. A clean
+// end ([DONE] or EOF without read errors, and no client disconnect) counts as
+// success and resets the cooldown.
 func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-chan types.UnifiedChunk, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
+	model := p.ModelFor(req.Model)
 	req.Stream = true
 	body, err := marshalTransform(p.vm, p.adapter, "transform_request", req)
 	if err != nil {
@@ -432,14 +639,14 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
 	ch := make(chan types.UnifiedChunk, 64)
 	sel := <-rc
 	if sel.err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		p.Release()
 		return nil, sel.err
 	}
 	if sel.resp.StatusCode != 200 {
 		raw, _ := io.ReadAll(sel.resp.Body)
 		sel.resp.Body.Close()
-		p.ReportStatus(sel.resp.StatusCode)
+		p.ReportStatus(model, sel.resp.StatusCode)
 		p.Release()
 		return nil, fmt.Errorf("api error %d: %s", sel.resp.StatusCode, truncate(string(raw), 500))
 	}
@@ -485,16 +692,25 @@ func (p *Provider) ChatStream(ctx context.Context, req *types.ChatRequest) (<-ch
 				return
 			}
 		}
+		// The stream ended cleanly ([DONE] seen or EOF without an upstream
+		// read error): record success so a previously cooled model can be
+		// retried. A client disconnect or mid-stream read error is neither
+		// success nor failure for scheduling purposes.
+		if ctx.Err() == nil && scanner.Err() == nil {
+			p.RecordSuccess(model)
+		}
 	}()
 	return ch, nil
 }
 
-// Image generates images via /v1/images/generations.
+// Image generates images via /v1/images/generations. Same scheduling-state
+// accounting as Chat: fail fast on busy, record per (source, model).
 func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*types.UnifiedResponse, error) {
-	if err := p.Acquire(ctx); err != nil {
+	if err := p.TryAcquire(ctx); err != nil {
 		return nil, err
 	}
 	defer p.Release()
+	model := p.ModelFor(req.Model)
 
 	b, _ := json.Marshal(req)
 	transformed, err := p.vm.Transform(p.adapter+"_image", "transform_request", string(b))
@@ -508,11 +724,11 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 	}
 	raw, status, err := p.do(ctx, p.ImageURL(), transformed, hdrs)
 	if err != nil {
-		p.reportError()
+		p.RecordFailure(model, 0)
 		return nil, err
 	}
 	if status != 200 {
-		p.ReportStatus(status)
+		p.ReportStatus(model, status)
 		return nil, fmt.Errorf("image api error %d: %s", status, truncate(raw, 500))
 	}
 	var out types.UnifiedResponse
@@ -520,7 +736,7 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 	unified, terr := p.vm.Transform(p.adapter+"_image", "transform_response", raw)
 	if terr == nil && unified != raw {
 		if err := json.Unmarshal([]byte(unified), &out); err == nil {
-			p.reportOK()
+			p.RecordSuccess(model)
 			return &out, nil
 		}
 	}
@@ -529,7 +745,7 @@ func (p *Provider) Image(ctx context.Context, req *types.ImageGenRequest) (*type
 		return nil, fmt.Errorf("unmarshal image response: %w", err)
 	}
 	out.ImageData = img.Data
-	p.reportOK()
+	p.RecordSuccess(model)
 	return &out, nil
 }
 
@@ -593,4 +809,4 @@ func truncate(s string, n int) string {
 		return s
 	}
 	return s[:n] + "..."
-}
+}