diff --git a/config.example.yaml b/config.example.yaml index 33578ad..7be75cd 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -24,6 +24,10 @@ runtime_file: runtime.json # tls_cert_file: /etc/llmsproxy/tls/fullchain.pem # tls_key_file: /etc/llmsproxy/tls/privkey.pem +# 可选：对外暴露的 base URL，用于“连接配置”复制片段中的 url。 +# 留空默认根据请求自动推断（http/https + Host）。内网/反代后建议显式配置。 +# public_base_url: https://gw.example.com/v1 + # 全局并发上限（0 = 不限） max_concurrent: 0 diff --git a/internal/config/config.go b/internal/config/config.go index 91d6c0a..b9324f5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ type Config struct { MaxConcurrent int `yaml:"max_concurrent"` // global inflight cap, 0 = unlimited TLSCertFile string `yaml:"tls_cert_file,omitempty"` // PEM cert; when set together with tls_key_file, serve HTTPS TLSKeyFile string `yaml:"tls_key_file,omitempty"` // PEM private key + PublicBaseURL string `yaml:"public_base_url,omitempty"` // external base for generated config snippets; default inferred from request Sources []Source `yaml:"sources"` } diff --git a/internal/core/core.go b/internal/core/core.go index b4cb297..3b4f3a5 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -149,6 +149,10 @@ func (c *Core) TLS() (cert, key string) { return c.cfg.TLSCertFile, c.cfg.TLSKeyFile } +// PublicBaseURL returns the externally advertised base used in generated +// connection snippets, or "" to infer it from the incoming request. +func (c *Core) PublicBaseURL() string { return c.cfg.PublicBaseURL } + // ---- gateway key management (web UI) ---- // ListKeys returns all gateway keys (admin view). diff --git a/internal/gateway/chat.go b/internal/gateway/chat.go index 3da2649..9b69927 100644 --- a/internal/gateway/chat.go +++ b/internal/gateway/chat.go @@ -228,7 +228,7 @@ func (g *Gateway) resolveByModel(model string) ([]*provider.Provider, string) { if isAuto(model) { return g.core.Registry().Resolve("AUTO"), "" } - return g.core.Registry().Resolve(model), model + return g.core.Registry().Resolve(model), g.core.Registry().EffectiveModel(model) } // chatOnly keeps providers that expose at least one chat-capable model, so a @@ -616,13 +616,17 @@ func (g *Gateway) streamChat(w http.ResponseWriter, ctx context.Context, cands [ type autoPlan struct { p *provider.Provider model string + tier int quota int64 win int64 } // autoPlans builds the schedulable AUTO slots from the persisted rules. A // slot is schedulable while its model is available and (when quota > 0) the -// tokens used within its reset window are below the quota. +// tokens used within its reset window are below the quota. Slots are returned +// tiered high→low, and within the same tier the order is rotated round-robin +// so concurrent requests spread evenly across equal-priority sources (still +// with failover to the next slot if one errors). func (g *Gateway) autoPlans() []autoPlan { rules := g.core.AutoRules() if len(rules) == 0 { @@ -641,9 +645,35 @@ func (g *Gateway) autoPlans() []autoPlan { if e.TokenQuota > 0 && g.stats.WindowTokens(e.Model, e.Source, win) >= e.TokenQuota { continue } - plans = append(plans, autoPlan{p: p, model: e.Model, quota: e.TokenQuota, win: win}) + plans = append(plans, autoPlan{p: p, model: e.Model, tier: e.Tier, quota: e.TokenQuota, win: win}) } - return plans + return g.rotateSameTier(plans) +} + +// rotateSameTier reorders the leading plan of each consecutive same-tier run +// using a global round-robin counter, so requests distribute across +// equal-priority sources while preserving tier ordering and in-tier failover. +func (g *Gateway) rotateSameTier(plans []autoPlan) []autoPlan { + if len(plans) < 2 { + // allow single slot without varying + return plans + } + rot := int(g.autoRR.Add(1)) + out := make([]autoPlan, 0, len(plans)) + for i := 0; i < len(plans); { + j := i + for j < len(plans) && plans[j].tier == plans[i].tier { + j++ + } + run := plans[i:j] + if len(run) > 1 { + off := rot % len(run) + run = append(run[off:], run[:off]...) + } + out = append(out, run...) + i = j + } + return out } // singleChatAuto runs a non-streaming AUTO request slot by slot: each slot diff --git a/internal/gateway/server.go b/internal/gateway/server.go index a72aadf..66b2928 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -15,6 +15,7 @@ import ( "net/url" "strings" "sync" + "sync/atomic" "time" "llmsproxy/internal/config" @@ -31,6 +32,7 @@ type Gateway struct { stats *Stats probeMu sync.Mutex lastProbe time.Time + autoRR atomic.Uint64 } func New(c *core.Core, gatewayKeys []string) (*Gateway, error) { @@ -355,12 +357,16 @@ func (g *Gateway) handleStatusAPI(w http.ResponseWriter, r *http.Request) { if r.TLS != nil { scheme = "https" } + baseURL := g.core.PublicBaseURL() + if baseURL == "" { + baseURL = scheme + "://" + host + "/v1" + } writeJSON(w, http.StatusOK, map[string]interface{}{ "default_model": g.core.DefaultModel(), "models": g.core.Registry().ModelList(), "sources": g.core.Registry().Status(), "adapters": g.core.ListAdapters(), - "base_url": scheme + "://" + host + "/v1", + "base_url": baseURL, "gateway_keys": ks, }) } diff --git a/internal/gateway/ui/index.html b/internal/gateway/ui/index.html index eab19b5..ab14af4 100644 --- a/internal/gateway/ui/index.html +++ b/internal/gateway/ui/index.html @@ -616,6 +616,13 @@ function legacyCopy(txt) { /* ---------- status tab ---------- */ let statsTimerId = null; const fmtN = n => (n ?? 0).toLocaleString(); +const fmtTok = n => { + const v = +n || 0; + if (v >= 1e9) return (v / 1e9).toFixed(2).replace(/\.?0+$/, '') + 'B'; + if (v >= 1e6) return (v / 1e6).toFixed(1).replace(/\.0$/, '') + 'M'; + if (v >= 1e3) return (v / 1e3).toFixed(1).replace(/\.0$/, '') + 'K'; + return String(v); +}; const fmtMs = ms => (ms == null || ms < 0) ? '—' : (ms >= 1000 ? (ms / 1000).toFixed(1) + 's' : ms + 'ms'); const fmtLat = (sum, n) => (n > 0 ? Math.round(sum / n) : -1); const fmtTime = ts => { @@ -629,10 +636,10 @@ async function renderStatus() { const base = window._base = s.base_url || location.origin + '/v1'; const key = window._key = (s.gateway_keys && s.gateway_keys[0]) || ''; const srcRows = s.sources.map(x => - `${esc(x.name)}${esc(x.adapter)} + `${x.live_available ? `${t('online')}` : `${t('offline')}`} + ${esc(x.name)}${esc(x.adapter)}
${x.models.map(m => `${esc(m)}`).join('')}
${esc(x.base_url || '')} - ${x.live_available ? `${t('online')}` : `${t('offline')}`} ${x.max_concurrent}`).join(''); $('#tab-status').innerHTML = `
@@ -645,7 +652,7 @@ async function renderStatus() {

${t('srcTitle')} (${s.sources.length})

-
${srcRows || ``}
${t('tName')}${t('tAdapter')}${t('tModels')}${t('tURL')}${t('tConn')}${t('tConc')}
${t('srcEmpty')}
+
${srcRows || ``}
${t('tConn')}${t('tName')}${t('tAdapter')}${t('tModels')}${t('tURL')}${t('tConc')}
${t('srcEmpty')}

${t('dashModel')}

@@ -719,8 +726,8 @@ async function paintStats() { if (kpi) kpi.innerHTML = `
${t('kpiActive')}
${st.active || 0}
${statsKeyF ? esc(statsKeyF) : ''}
${t('kpiReqs')}
${fmtN(tot.reqs)}
${t('kpiOk')} ${okr}%
-
${t('kpiTokens')}
${fmtN(tot.tokens)}
-
${t('thPrompt')} ${fmtN(tot.prompt_tokens)} · ${t('thCompl')} ${fmtN(tot.completion_tokens)}
+
${t('kpiTokens')}
${fmtTok(tot.tokens)}
+
${t('thPrompt')} ${fmtTok(tot.prompt_tokens)} · ${t('thCompl')} ${fmtTok(tot.completion_tokens)}
${t('kpiLat')}
${fmtMs(avg)}
${t('kpiMaxLat')} ${fmtMs(tot.latency_max_ms)}
`; paintModelTable(st.by_model || []); paintSrcTable(st.by_source || []); @@ -740,7 +747,7 @@ function paintModelTable(rows) { return `${esc(r.name)} ${w.toFixed(0)}%
${fmtN(r.reqs)} ${fmtN(r.ok)}${fmtN(r.err)} - ${fmtN(r.prompt_tokens)}${fmtN(r.completion_tokens)} + ${fmtTok(r.prompt_tokens)}${fmtTok(r.completion_tokens)} ${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}${fmtMs(r.latency_max_ms)}`; }).join('') + '
'; } @@ -751,7 +758,7 @@ function paintSrcTable(rows) { ${t('thTokens')}${t('thAvgLat')}${t('thMaxLat')}` + rows.map(r => `${esc(r.name)} ${fmtN(r.reqs)}${fmtN(r.ok)}${fmtN(r.err)} - ${fmtN(r.tokens)} + ${fmtTok(r.tokens)} ${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}${fmtMs(r.latency_max_ms)}`).join('') + ''; } function paintKeyTable(rows, keyNames) { @@ -761,7 +768,7 @@ function paintKeyTable(rows, keyNames) { ${t('thTokens')}${t('thAvgLat')}` + rows.map(r => ` ${fmtN(r.reqs)}${fmtN(r.ok)}${fmtN(r.err)} - ${fmtN(r.tokens)}${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}`).join('') + ''; + ${fmtTok(r.tokens)}${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}`).join('') + ''; } function paintRecords(records, keyNames) { const el = $('#tb-recs'); if (!el) return; @@ -772,7 +779,7 @@ function paintRecords(records, keyNames) { ${fmtTime(r.time)} ${r.ok ? `${r.status || 200}` : `${r.status || 500}`} ${esc(keyNames[r.key] ? keyNames[r.key] + ' · ' + r.key : r.key)}${esc(r.type)}${esc(r.model)}${esc(r.source || '')} - ${fmtN(r.prompt_tokens)}${fmtN(r.completion_tokens)}${fmtMs(r.latency_ms)}`).join('') + ''; + ${fmtTok(r.prompt_tokens)}${fmtTok(r.completion_tokens)}${fmtMs(r.latency_ms)}`).join('') + ''; } function showModelConfig(srcName, model) { const el = $('#conncfg'); if (!el) return; @@ -1745,7 +1752,7 @@ function maybeWarnSeed(k) { const wrap = document.createElement('div'); wrap.id = 'modal-wrap'; wrap.style.cssText = 'position:fixed;inset:0;background:rgba(15,22,44,.45);display:flex;align-items:flex-start;justify-content:center;overflow:auto;padding:48px 20px;z-index:50'; wrap.innerHTML = `
-

⚠ ${esc(t('seedWarnTitle'))}

+

${esc(t('seedWarnTitle'))}

${esc(t('seedWarnText'))}

diff --git a/internal/lua/vm_test.go b/internal/lua/vm_test.go index 5bde928..78bc60d 100644 --- a/internal/lua/vm_test.go +++ b/internal/lua/vm_test.go @@ -2,12 +2,20 @@ package lua import ( "encoding/json" + "path/filepath" "strings" "testing" ) +// freshAdapterDir returns a path under a temp dir that does not exist yet, so +// VM.Start() treats it as first-run and seeds the bundled adapters. +func freshAdapterDir(t *testing.T) string { + t.Helper() + return filepath.Join(t.TempDir(), "adapters") +} + func TestLoadBundledAdapters(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatalf("start: %v", err) } @@ -28,7 +36,7 @@ func TestLoadBundledAdapters(t *testing.T) { } func TestTransformRequest(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatal(err) } @@ -43,7 +51,7 @@ func TestTransformRequest(t *testing.T) { } func TestBuildHeadersFallbackStatic(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatal(err) } @@ -58,7 +66,7 @@ func TestBuildHeadersFallbackStatic(t *testing.T) { } func TestBuildHeadersCustomHook(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatal(err) } @@ -88,7 +96,7 @@ func TestBuildHeadersCustomHook(t *testing.T) { } func TestDisableThinkingPassthrough(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatal(err) } @@ -130,7 +138,7 @@ func TestDisableThinkingPassthrough(t *testing.T) { } func TestMultimodalTransform(t *testing.T) { - vm := NewVM(t.TempDir()) + vm := NewVM(freshAdapterDir(t)) if err := vm.Start(); err != nil { t.Fatal(err) } diff --git a/internal/provider/provider_test.go b/internal/provider/provider_test.go index 8587ac5..d24c548 100644 --- a/internal/provider/provider_test.go +++ b/internal/provider/provider_test.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "path/filepath" "sync" "testing" "time" @@ -17,7 +18,7 @@ import ( func newTestProvider(t *testing.T, src config.Source) *Provider { t.Helper() - vm := lua.NewVM(t.TempDir()) + vm := lua.NewVM(filepath.Join(t.TempDir(), "adapters")) if err := vm.Start(); err != nil { t.Fatalf("vm: %v", err) } diff --git a/internal/provider/registry.go b/internal/provider/registry.go index 9d7c106..3990a34 100644 --- a/internal/provider/registry.go +++ b/internal/provider/registry.go @@ -125,10 +125,55 @@ func (r *Registry) Resolve(model string) []*Provider { // switch to the owning source but pin the model via request return []*Provider{p} } + // "source-model" / "source:model" / "source/model" pinning — disambiguates + // duplicate model ids across sources. + if p := r.ResolvePinned(model); p != nil { + return []*Provider{p} + } // unknown model -> fall back to default/AUTO chain return r.AUTOChain() } +// EffectiveModel strips a "source-model" / "source:model" / "source/model" +// pinning prefix and returns the bare model id when that source serves it; +// otherwise it returns the input unchanged. +func (r *Registry) EffectiveModel(model string) string { + sep := strings.IndexAny(model, "-:/") + if sep < 1 || sep == len(model)-1 { + return model + } + src, m := model[:sep], model[sep+1:] + r.mu.RLock() + defer r.mu.RUnlock() + for _, p := range r.providers { + if strings.EqualFold(p.Name(), src) && p.ModelByID(m) != nil { + return m + } + } + return model +} + +// ResolvePinned resolves "source-model" / "source:model" / "source/model" to +// the exact source, or nil if the source does not serve that model. +func (r *Registry) ResolvePinned(model string) *Provider { + sep := strings.IndexAny(model, "-:/") + if sep < 1 || sep == len(model)-1 { + return nil + } + src, m := model[:sep], model[sep+1:] + r.mu.RLock() + defer r.mu.RUnlock() + for _, p := range r.providers { + if !strings.EqualFold(p.Name(), src) { + continue + } + if p.ModelByID(m) != nil { + return p + } + } + return nil +} + // AUTOChain returns the priority-sorted providers for AUTO. func (r *Registry) AUTOChain() []*Provider { return r.Resolve("AUTO")