// Package scheduler implements request scheduling across providers: per-source // concurrency caps (acquire with wait = queuing), AUTO model fallback chains, // and exponential backoff via provider health. package scheduler import ( "context" "fmt" "llmsproxy/internal/provider" "llmsproxy/internal/types" ) // Scheduler drives one chat tool call across the candidate provider chain. type Scheduler struct { // MaxRetries how many fallback providers to try before failing. MaxRetries int } func New(maxRetries int) *Scheduler { if maxRetries < 0 { maxRetries = 0 } return &Scheduler{MaxRetries: maxRetries} } // Provider is the minimal interface the scheduler needs to schedule over. type Provider interface { Name() string Available() bool Chat(ctx context.Context, req *types.ChatRequest) (*types.UnifiedResponse, error) ChatStream(ctx context.Context, req *types.ChatRequest) (<-chan types.UnifiedChunk, error) Image(ctx context.Context, req *types.ImageGenRequest) (*types.UnifiedResponse, error) } // FromRegistry converts *provider.Provider slices to the scheduler interface. func FromRegistry(ps []*provider.Provider) []Provider { out := make([]Provider, len(ps)) for i, p := range ps { out[i] = p } return out } // Chat runs a chat request across cands, falling back on failure. func (s *Scheduler) Chat(ctx context.Context, cands []Provider, req *types.ChatRequest) (*types.UnifiedResponse, error) { attempts := s.MaxRetries + 1 var lastErr error for i := 0; i < attempts && i < len(cands); i++ { p := cands[i] resp, err := p.Chat(ctx, req) if ctx.Err() != nil { return nil, ctx.Err() } if err == nil { return resp, nil } lastErr = fmt.Errorf("provider %s: %w", p.Name(), err) } if lastErr == nil { // if loop couldn't run because cands was short but no error recorded yet if len(cands) == 0 { return nil, fmt.Errorf("no provider available") } // should not happen return nil, lastErr } return nil, lastErr } // ChatStream runs a streaming chat across cands, falling back early on connect errors. func (s *Scheduler) ChatStream(ctx context.Context, cands []Provider, req *types.ChatRequest) (<-chan types.UnifiedChunk, error) { attempts := s.MaxRetries + 1 var lastErr error for i := 0; i < attempts && i < len(cands); i++ { p := cands[i] resp, err := p.ChatStream(ctx, req) if err == nil { return resp, nil } lastErr = fmt.Errorf("provider %s: %w", p.Name(), err) } if lastErr == nil { if len(cands) == 0 { return nil, fmt.Errorf("no provider available") } } return nil, lastErr } // Image runs an image-generation request across cands. func (s *Scheduler) Image(ctx context.Context, cands []Provider, req *types.ImageGenRequest) (*types.UnifiedResponse, error) { attempts := s.MaxRetries + 1 var lastErr error for i := 0; i < attempts && i < len(cands); i++ { p := cands[i] resp, err := p.Image(ctx, req) if err == nil { return resp, nil } lastErr = fmt.Errorf("provider %s: %w", p.Name(), err) } if lastErr == nil && len(cands) == 0 { return nil, fmt.Errorf("no provider available") } return nil, lastErr }