mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-19 16:39:15 +00:00
feat: source-model routing disambiguation (source-model/:/ prefix); same-tier round-robin load balancing; public_base_url for copy config; fmtTok(B/M/K) unit scaling; status column reorder (reachability first); drop emoji from seed-warn modal; fix tests for first-run adapter seeding; install lua5.1 dev lib
This commit is contained in:
root
@ -24,6 +24,10 @@ runtime_file: runtime.json
|
||||
# tls_cert_file: /etc/llmsproxy/tls/fullchain.pem
|
||||
# tls_key_file: /etc/llmsproxy/tls/privkey.pem
|
||||
|
||||
# 可选：对外暴露的 base URL，用于“连接配置”复制片段中的 url。
|
||||
# 留空默认根据请求自动推断（http/https + Host）。内网/反代后建议显式配置。
|
||||
# public_base_url: https://gw.example.com/v1
|
||||
|
||||
# 全局并发上限（0 = 不限）
|
||||
max_concurrent: 0
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ type Config struct {
|
||||
MaxConcurrent int `yaml:"max_concurrent"` // global inflight cap, 0 = unlimited
|
||||
TLSCertFile string `yaml:"tls_cert_file,omitempty"` // PEM cert; when set together with tls_key_file, serve HTTPS
|
||||
TLSKeyFile string `yaml:"tls_key_file,omitempty"` // PEM private key
|
||||
PublicBaseURL string `yaml:"public_base_url,omitempty"` // external base for generated config snippets; default inferred from request
|
||||
Sources []Source `yaml:"sources"`
|
||||
}
|
||||
|
||||
|
||||
@ -149,6 +149,10 @@ func (c *Core) TLS() (cert, key string) {
|
||||
return c.cfg.TLSCertFile, c.cfg.TLSKeyFile
|
||||
}
|
||||
|
||||
// PublicBaseURL returns the externally advertised base used in generated
|
||||
// connection snippets, or "" to infer it from the incoming request.
|
||||
func (c *Core) PublicBaseURL() string { return c.cfg.PublicBaseURL }
|
||||
|
||||
// ---- gateway key management (web UI) ----
|
||||
|
||||
// ListKeys returns all gateway keys (admin view).
|
||||
|
||||
@ -228,7 +228,7 @@ func (g *Gateway) resolveByModel(model string) ([]*provider.Provider, string) {
|
||||
if isAuto(model) {
|
||||
return g.core.Registry().Resolve("AUTO"), ""
|
||||
}
|
||||
return g.core.Registry().Resolve(model), model
|
||||
return g.core.Registry().Resolve(model), g.core.Registry().EffectiveModel(model)
|
||||
}
|
||||
|
||||
// chatOnly keeps providers that expose at least one chat-capable model, so a
|
||||
@ -616,13 +616,17 @@ func (g *Gateway) streamChat(w http.ResponseWriter, ctx context.Context, cands [
|
||||
type autoPlan struct {
|
||||
p *provider.Provider
|
||||
model string
|
||||
tier int
|
||||
quota int64
|
||||
win int64
|
||||
}
|
||||
|
||||
// autoPlans builds the schedulable AUTO slots from the persisted rules. A
|
||||
// slot is schedulable while its model is available and (when quota > 0) the
|
||||
// tokens used within its reset window are below the quota.
|
||||
// tokens used within its reset window are below the quota. Slots are returned
|
||||
// tiered high→low, and within the same tier the order is rotated round-robin
|
||||
// so concurrent requests spread evenly across equal-priority sources (still
|
||||
// with failover to the next slot if one errors).
|
||||
func (g *Gateway) autoPlans() []autoPlan {
|
||||
rules := g.core.AutoRules()
|
||||
if len(rules) == 0 {
|
||||
@ -641,9 +645,35 @@ func (g *Gateway) autoPlans() []autoPlan {
|
||||
if e.TokenQuota > 0 && g.stats.WindowTokens(e.Model, e.Source, win) >= e.TokenQuota {
|
||||
continue
|
||||
}
|
||||
plans = append(plans, autoPlan{p: p, model: e.Model, quota: e.TokenQuota, win: win})
|
||||
plans = append(plans, autoPlan{p: p, model: e.Model, tier: e.Tier, quota: e.TokenQuota, win: win})
|
||||
}
|
||||
return plans
|
||||
return g.rotateSameTier(plans)
|
||||
}
|
||||
|
||||
// rotateSameTier reorders the leading plan of each consecutive same-tier run
|
||||
// using a global round-robin counter, so requests distribute across
|
||||
// equal-priority sources while preserving tier ordering and in-tier failover.
|
||||
func (g *Gateway) rotateSameTier(plans []autoPlan) []autoPlan {
|
||||
if len(plans) < 2 {
|
||||
// allow single slot without varying
|
||||
return plans
|
||||
}
|
||||
rot := int(g.autoRR.Add(1))
|
||||
out := make([]autoPlan, 0, len(plans))
|
||||
for i := 0; i < len(plans); {
|
||||
j := i
|
||||
for j < len(plans) && plans[j].tier == plans[i].tier {
|
||||
j++
|
||||
}
|
||||
run := plans[i:j]
|
||||
if len(run) > 1 {
|
||||
off := rot % len(run)
|
||||
run = append(run[off:], run[:off]...)
|
||||
}
|
||||
out = append(out, run...)
|
||||
i = j
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// singleChatAuto runs a non-streaming AUTO request slot by slot: each slot
|
||||
|
||||
@ -15,6 +15,7 @@ import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"llmsproxy/internal/config"
|
||||
@ -31,6 +32,7 @@ type Gateway struct {
|
||||
stats *Stats
|
||||
probeMu sync.Mutex
|
||||
lastProbe time.Time
|
||||
autoRR atomic.Uint64
|
||||
}
|
||||
|
||||
func New(c *core.Core, gatewayKeys []string) (*Gateway, error) {
|
||||
@ -355,12 +357,16 @@ func (g *Gateway) handleStatusAPI(w http.ResponseWriter, r *http.Request) {
|
||||
if r.TLS != nil {
|
||||
scheme = "https"
|
||||
}
|
||||
baseURL := g.core.PublicBaseURL()
|
||||
if baseURL == "" {
|
||||
baseURL = scheme + "://" + host + "/v1"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"default_model": g.core.DefaultModel(),
|
||||
"models": g.core.Registry().ModelList(),
|
||||
"sources": g.core.Registry().Status(),
|
||||
"adapters": g.core.ListAdapters(),
|
||||
"base_url": scheme + "://" + host + "/v1",
|
||||
"base_url": baseURL,
|
||||
"gateway_keys": ks,
|
||||
})
|
||||
}
|
||||
|
||||
@ -616,6 +616,13 @@ function legacyCopy(txt) {
|
||||
/* ---------- status tab ---------- */
|
||||
let statsTimerId = null;
|
||||
const fmtN = n => (n ?? 0).toLocaleString();
|
||||
const fmtTok = n => {
|
||||
const v = +n || 0;
|
||||
if (v >= 1e9) return (v / 1e9).toFixed(2).replace(/\.?0+$/, '') + 'B';
|
||||
if (v >= 1e6) return (v / 1e6).toFixed(1).replace(/\.0$/, '') + 'M';
|
||||
if (v >= 1e3) return (v / 1e3).toFixed(1).replace(/\.0$/, '') + 'K';
|
||||
return String(v);
|
||||
};
|
||||
const fmtMs = ms => (ms == null || ms < 0) ? '—' : (ms >= 1000 ? (ms / 1000).toFixed(1) + 's' : ms + 'ms');
|
||||
const fmtLat = (sum, n) => (n > 0 ? Math.round(sum / n) : -1);
|
||||
const fmtTime = ts => {
|
||||
@ -629,10 +636,10 @@ async function renderStatus() {
|
||||
const base = window._base = s.base_url || location.origin + '/v1';
|
||||
const key = window._key = (s.gateway_keys && s.gateway_keys[0]) || '';
|
||||
const srcRows = s.sources.map(x =>
|
||||
`<tr><td><b>${esc(x.name)}</b></td><td>${esc(x.adapter)}</td>
|
||||
`<tr><td>${x.live_available ? `<span class="tag tag-green"><i class="net-dot"></i>${t('online')}</span>` : `<span class="tag tag-red" title="${esc(x.last_error || '')}"><i class="net-dot"></i>${t('offline')}</span>`}</td>
|
||||
<td><b>${esc(x.name)}</b></td><td>${esc(x.adapter)}</td>
|
||||
<td><div class="src-models">${x.models.map(m => `<span class="tag tag-blue modelchip" onclick="showModelConfig('${escAttr(x.name)}','${escAttr(m)}')">${esc(m)}</span>`).join('')}</div></td>
|
||||
<td><span class="muted">${esc(x.base_url || '')}</span></td>
|
||||
<td>${x.live_available ? `<span class="tag tag-green"><i class="net-dot"></i>${t('online')}</span>` : `<span class="tag tag-red" title="${esc(x.last_error || '')}"><i class="net-dot"></i>${t('offline')}</span>`}</td>
|
||||
<td>${x.max_concurrent}</td></tr>`).join('');
|
||||
$('#tab-status').innerHTML = `
|
||||
<div class="kpis" id="kpi-row"></div>
|
||||
@ -645,7 +652,7 @@ async function renderStatus() {
|
||||
<div class="chips" id="model-chips"></div>
|
||||
</div>
|
||||
<div class="card"><h2>${t('srcTitle')} (${s.sources.length})</h2>
|
||||
<div class="tbl-wrap"><table><tr><th>${t('tName')}</th><th>${t('tAdapter')}</th><th>${t('tModels')}</th><th>${t('tURL')}</th><th>${t('tConn')}</th><th>${t('tConc')}</th></tr>${srcRows || `<tr><td colspan="6" class="empty">${t('srcEmpty')}</td></tr>`}</table></div>
|
||||
<div class="tbl-wrap"><table><tr><th>${t('tConn')}</th><th>${t('tName')}</th><th>${t('tAdapter')}</th><th>${t('tModels')}</th><th>${t('tURL')}</th><th>${t('tConc')}</th></tr>${srcRows || `<tr><td colspan="6" class="empty">${t('srcEmpty')}</td></tr>`}</table></div>
|
||||
</div>
|
||||
<div class="dash-row">
|
||||
<div class="card"><h2>${t('dashModel')}</h2><div id="tb-model"></div></div>
|
||||
@ -719,8 +726,8 @@ async function paintStats() {
|
||||
if (kpi) kpi.innerHTML = `
|
||||
<div class="kpi"><div class="k-lab">${t('kpiActive')} <span class="dot"></span></div><div class="k-val">${st.active || 0}</div><div class="k-sub">${statsKeyF ? esc(statsKeyF) : ''}</div></div>
|
||||
<div class="kpi"><div class="k-lab">${t('kpiReqs')}</div><div class="k-val">${fmtN(tot.reqs)}</div><div class="k-sub">${t('kpiOk')} <b class="${okr >= 90 ? 'okc' : 'errc'}">${okr}%</b></div></div>
|
||||
<div class="kpi"><div class="k-lab">${t('kpiTokens')}</div><div class="k-val">${fmtN(tot.tokens)}</div>
|
||||
<div class="k-sub">${t('thPrompt')} ${fmtN(tot.prompt_tokens)} · ${t('thCompl')} ${fmtN(tot.completion_tokens)}</div></div>
|
||||
<div class="kpi"><div class="k-lab">${t('kpiTokens')}</div><div class="k-val">${fmtTok(tot.tokens)}</div>
|
||||
<div class="k-sub">${t('thPrompt')} ${fmtTok(tot.prompt_tokens)} · ${t('thCompl')} ${fmtTok(tot.completion_tokens)}</div></div>
|
||||
<div class="kpi"><div class="k-lab">${t('kpiLat')}</div><div class="k-val">${fmtMs(avg)}</div><div class="k-sub">${t('kpiMaxLat')} ${fmtMs(tot.latency_max_ms)}</div></div>`;
|
||||
paintModelTable(st.by_model || []);
|
||||
paintSrcTable(st.by_source || []);
|
||||
@ -740,7 +747,7 @@ function paintModelTable(rows) {
|
||||
return `<tr><td>${esc(r.name)}</td>
|
||||
<td class="num"><span class="mini">${w.toFixed(0)}%</span><div class="bar"><i style="width:${w}%"></i></div>${fmtN(r.reqs)}</td>
|
||||
<td class="num okc">${fmtN(r.ok)}</td><td class="num errc">${fmtN(r.err)}</td>
|
||||
<td class="num">${fmtN(r.prompt_tokens)}</td><td class="num">${fmtN(r.completion_tokens)}</td>
|
||||
<td class="num">${fmtTok(r.prompt_tokens)}</td><td class="num">${fmtTok(r.completion_tokens)}</td>
|
||||
<td class="num">${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}</td><td class="num">${fmtMs(r.latency_max_ms)}</td></tr>`;
|
||||
}).join('') + '</table></div>';
|
||||
}
|
||||
@ -751,7 +758,7 @@ function paintSrcTable(rows) {
|
||||
<th class="num">${t('thTokens')}</th><th class="num">${t('thAvgLat')}</th><th class="num">${t('thMaxLat')}</th></tr>` +
|
||||
rows.map(r => `<tr><td><b>${esc(r.name)}</b></td>
|
||||
<td class="num">${fmtN(r.reqs)}</td><td class="num okc">${fmtN(r.ok)}</td><td class="num errc">${fmtN(r.err)}</td>
|
||||
<td class="num">${fmtN(r.tokens)}</td>
|
||||
<td class="num">${fmtTok(r.tokens)}</td>
|
||||
<td class="num">${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}</td><td class="num">${fmtMs(r.latency_max_ms)}</td></tr>`).join('') + '</table></div>';
|
||||
}
|
||||
function paintKeyTable(rows, keyNames) {
|
||||
@ -761,7 +768,7 @@ function paintKeyTable(rows, keyNames) {
|
||||
<th class="num">${t('thTokens')}</th><th class="num">${t('thAvgLat')}</th></tr>` +
|
||||
rows.map(r => `<tr><td><button class="ghost small" onclick="renderKeyF('${escAttr(r.name)}')">${esc(keyNames[r.name] ? keyNames[r.name] + ' · ' + r.name : r.name)}</button></td>
|
||||
<td class="num">${fmtN(r.reqs)}</td><td class="num okc">${fmtN(r.ok)}</td><td class="num errc">${fmtN(r.err)}</td>
|
||||
<td class="num">${fmtN(r.tokens)}</td><td class="num">${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}</td></tr>`).join('') + '</table></div>';
|
||||
<td class="num">${fmtTok(r.tokens)}</td><td class="num">${fmtMs(fmtLat(r.latency_sum_ms, r.reqs))}</td></tr>`).join('') + '</table></div>';
|
||||
}
|
||||
function paintRecords(records, keyNames) {
|
||||
const el = $('#tb-recs'); if (!el) return;
|
||||
@ -772,7 +779,7 @@ function paintRecords(records, keyNames) {
|
||||
<td class="t-tag">${fmtTime(r.time)}</td>
|
||||
<td class="num">${r.ok ? `<span class="tag tag-green">${r.status || 200}</span>` : `<span class="tag tag-red" title="${esc(r.error || '')}">${r.status || 500}</span>`}</td>
|
||||
<td>${esc(keyNames[r.key] ? keyNames[r.key] + ' · ' + r.key : r.key)}</td><td class="t-tag">${esc(r.type)}</td><td>${esc(r.model)}</td><td>${esc(r.source || '')}</td>
|
||||
<td class="num">${fmtN(r.prompt_tokens)}</td><td class="num">${fmtN(r.completion_tokens)}</td><td class="num">${fmtMs(r.latency_ms)}</td></tr>`).join('') + '</table>';
|
||||
<td class="num">${fmtTok(r.prompt_tokens)}</td><td class="num">${fmtTok(r.completion_tokens)}</td><td class="num">${fmtMs(r.latency_ms)}</td></tr>`).join('') + '</table>';
|
||||
}
|
||||
function showModelConfig(srcName, model) {
|
||||
const el = $('#conncfg'); if (!el) return;
|
||||
@ -1745,7 +1752,7 @@ function maybeWarnSeed(k) {
|
||||
const wrap = document.createElement('div'); wrap.id = 'modal-wrap';
|
||||
wrap.style.cssText = 'position:fixed;inset:0;background:rgba(15,22,44,.45);display:flex;align-items:flex-start;justify-content:center;overflow:auto;padding:48px 20px;z-index:50';
|
||||
wrap.innerHTML = `<div class="card" style="width:460px;max-width:100%">
|
||||
<h2>⚠ ${esc(t('seedWarnTitle'))}</h2>
|
||||
<h2>${esc(t('seedWarnTitle'))}</h2>
|
||||
<p class="muted" style="line-height:1.6">${esc(t('seedWarnText'))}</p>
|
||||
<p><button onclick="this.closest('#modal-wrap').remove();goTab('keys')">${esc(t('seedWarnGo'))}</button>
|
||||
<button class="ghost" onclick="this.closest('#modal-wrap').remove()">${esc(t('seedWarnLater'))}</button>
|
||||
|
||||
@ -2,12 +2,20 @@ package lua
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// freshAdapterDir returns a path under a temp dir that does not exist yet, so
|
||||
// VM.Start() treats it as first-run and seeds the bundled adapters.
|
||||
func freshAdapterDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
return filepath.Join(t.TempDir(), "adapters")
|
||||
}
|
||||
|
||||
func TestLoadBundledAdapters(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatalf("start: %v", err)
|
||||
}
|
||||
@ -28,7 +36,7 @@ func TestLoadBundledAdapters(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTransformRequest(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -43,7 +51,7 @@ func TestTransformRequest(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildHeadersFallbackStatic(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -58,7 +66,7 @@ func TestBuildHeadersFallbackStatic(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestBuildHeadersCustomHook(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -88,7 +96,7 @@ func TestBuildHeadersCustomHook(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestDisableThinkingPassthrough(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -130,7 +138,7 @@ func TestDisableThinkingPassthrough(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMultimodalTransform(t *testing.T) {
|
||||
vm := NewVM(t.TempDir())
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
@ -17,7 +18,7 @@ import (
|
||||
|
||||
func newTestProvider(t *testing.T, src config.Source) *Provider {
|
||||
t.Helper()
|
||||
vm := lua.NewVM(t.TempDir())
|
||||
vm := lua.NewVM(filepath.Join(t.TempDir(), "adapters"))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatalf("vm: %v", err)
|
||||
}
|
||||
|
||||
@ -125,10 +125,55 @@ func (r *Registry) Resolve(model string) []*Provider {
|
||||
// switch to the owning source but pin the model via request
|
||||
return []*Provider{p}
|
||||
}
|
||||
// "source-model" / "source:model" / "source/model" pinning — disambiguates
|
||||
// duplicate model ids across sources.
|
||||
if p := r.ResolvePinned(model); p != nil {
|
||||
return []*Provider{p}
|
||||
}
|
||||
// unknown model -> fall back to default/AUTO chain
|
||||
return r.AUTOChain()
|
||||
}
|
||||
|
||||
// EffectiveModel strips a "source-model" / "source:model" / "source/model"
|
||||
// pinning prefix and returns the bare model id when that source serves it;
|
||||
// otherwise it returns the input unchanged.
|
||||
func (r *Registry) EffectiveModel(model string) string {
|
||||
sep := strings.IndexAny(model, "-:/")
|
||||
if sep < 1 || sep == len(model)-1 {
|
||||
return model
|
||||
}
|
||||
src, m := model[:sep], model[sep+1:]
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, p := range r.providers {
|
||||
if strings.EqualFold(p.Name(), src) && p.ModelByID(m) != nil {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return model
|
||||
}
|
||||
|
||||
// ResolvePinned resolves "source-model" / "source:model" / "source/model" to
|
||||
// the exact source, or nil if the source does not serve that model.
|
||||
func (r *Registry) ResolvePinned(model string) *Provider {
|
||||
sep := strings.IndexAny(model, "-:/")
|
||||
if sep < 1 || sep == len(model)-1 {
|
||||
return nil
|
||||
}
|
||||
src, m := model[:sep], model[sep+1:]
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
for _, p := range r.providers {
|
||||
if !strings.EqualFold(p.Name(), src) {
|
||||
continue
|
||||
}
|
||||
if p.ModelByID(m) != nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AUTOChain returns the priority-sorted providers for AUTO.
|
||||
func (r *Registry) AUTOChain() []*Provider {
|
||||
return r.Resolve("AUTO")
|
||||
|
||||
Reference in New Issue
Block a user