From 854b3e2e373972619c1cd54bc0c1b407610ab77f Mon Sep 17 00:00:00 2001 From: root Date: Tue, 11 Aug 2026 13:36:23 +0800 Subject: [PATCH] fix: AUTO tier order ascending + stats/CSV export completeness + UI key-view toggle & exit affordance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scheduler: BuildChain sorts tiers ascending so tier 1 (highest priority) is tried first; previously descending inverted the chain (P10-1..P10-5) - stats/api: handleStatsAPI limit→20000; CSV reads full audit via new Stats.AuditRecords (includes *.old rotation); key_names mapped by keyID() masked key; non-admin filter and keys-csv use masked keys - server: ring buffer NewStats(10000) - ui: dash-row card scroll area moved to table container (fix overflow below card); key view toggle (re-click same key returns to global) + kpi-exit affordance; rec-exit span - chat: chain-failure record now surfaces first failed tier; AUTO comment sync --- internal/gateway/api.go | 24 +++++++++------ internal/gateway/chat.go | 9 +++--- internal/gateway/server.go | 2 +- internal/gateway/stats.go | 45 ++++++++++++++++++++++++++++ internal/gateway/ui/index.html | 29 +++++++++++------- internal/scheduler/scheduler.go | 24 +++++++-------- internal/scheduler/scheduler_test.go | 14 ++++----- 7 files changed, 104 insertions(+), 43 deletions(-) diff --git a/internal/gateway/api.go b/internal/gateway/api.go index 601b8fe..7cb0da5 100644 --- a/internal/gateway/api.go +++ b/internal/gateway/api.go @@ -130,14 +130,15 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) { } limit := 500 if v := r.URL.Query().Get("limit"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 5000 { + if n, err := strconv.Atoi(v); err == nil && n > 0 && n <= 20000 { limit = n } } key := r.URL.Query().Get("key") if reqRole(r.Context()) != "admin" { - // user keys may only see their own usage - use full key for internal filtering - key = reqKey(r.Context()) + // user keys may only see their own usage - records and aggregates are + // keyed by the masked id (keyID), so filter on that masked form. + key = keyID(reqKey(r.Context())) } if r.URL.Query().Get("export") == "csv" { from, _ := strconv.ParseInt(r.URL.Query().Get("from"), 10, 64) @@ -150,10 +151,10 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) { cw := csv.NewWriter(w) names := map[string]string{} for _, k := range g.core.ListKeys() { - names[k.Key] = k.Name + names[keyID(k.Key)] = k.Name } _ = cw.Write([]string{"time", "key", "key_name", "type", "model", "source", "status", "ok", "prompt_tokens", "completion_tokens", "latency_ms", "error"}) - for _, rec := range g.stats.Records(from, to, key) { + for _, rec := range g.stats.AuditRecords(from, to, key) { _ = cw.Write([]string{ time.UnixMilli(rec.Time).Format(time.RFC3339), rec.Key, @@ -180,7 +181,13 @@ if r.URL.Query().Get("export") == "keys-csv" { // Use the same key filtering as the JSON API exportKey := r.URL.Query().Get("key") if reqRole(r.Context()) != "admin" { - exportKey = reqKey(r.Context()) + exportKey = keyID(reqKey(r.Context())) + } + // by_key rows are keyed by the masked id (keyID); build a masked-id -> + // record map so names/roles/models resolve for the export. + byMasked := map[string]config.GWKey{} + for _, k := range g.core.ListKeys() { + byMasked[keyID(k.Key)] = k } snap := g.stats.Snapshot(0, exportKey) byKeyRaw, ok := snap["by_key"].([]StatsRow) @@ -188,8 +195,7 @@ if r.URL.Query().Get("export") == "keys-csv" { byKeyRaw = []StatsRow{} } for _, row := range byKeyRaw { - key := row.Name - keyInfo, found := g.core.FindKey(key) + keyInfo, found := byMasked[row.Name] name := "" role := "" models := "" @@ -237,7 +243,7 @@ if r.URL.Query().Get("export") == "keys-csv" { snap := g.stats.Snapshot(limit, key) keyNames := map[string]string{} for _, k := range g.core.ListKeys() { - keyNames[k.Key] = k.Name + keyNames[keyID(k.Key)] = k.Name } snap["key_names"] = keyNames writeJSON(w, http.StatusOK, snap) diff --git a/internal/gateway/chat.go b/internal/gateway/chat.go index 6ccceac..c1a4e16 100644 --- a/internal/gateway/chat.go +++ b/internal/gateway/chat.go @@ -648,10 +648,11 @@ func (g *Gateway) streamChat(w http.ResponseWriter, ctx context.Context, cands [ } // singleChatAuto runs a non-streaming AUTO request down the chain (see -// scheduler.ChainChat): tiers descending, per-tier round-robin ordered by -// preference, cooldown as the only hard skip, busy slots skipped without -// penalty and a bounded busy wait. When every tier fails, the response is a -// 503 carrying the per-tier error summary (which source/model failed why). +// scheduler.ChainChat): tiers ascending (tier 1 = highest priority first), +// per-tier round-robin ordered by preference, cooldown as the only hard skip, +// busy slots skipped without penalty and a bounded busy wait. When every +// tier fails, the response is a 503 carrying the per-tier error summary +// (which source/model failed why). func (g *Gateway) singleChatAuto(w http.ResponseWriter, ctx context.Context, chain *scheduler.Chain, req *types.ChatRequest, rec *Req, quotaExhausted func(*scheduler.Slot) bool) { rec.LatMs = 0 t0 := time.Now() diff --git a/internal/gateway/server.go b/internal/gateway/server.go index 99837b9..9bd591b 100644 --- a/internal/gateway/server.go +++ b/internal/gateway/server.go @@ -38,7 +38,7 @@ func New(c *core.Core, gatewayKeys []string) (*Gateway, error) { if err != nil { return nil, err } - st := NewStats(3000) + st := NewStats(10000) if cfg := c.Config(); cfg != nil && cfg.RuntimeFile != "" { st.LoadAudit(cfg.RuntimeFile + ".audit.jsonl") } diff --git a/internal/gateway/stats.go b/internal/gateway/stats.go index ef338eb..eec5d74 100644 --- a/internal/gateway/stats.go +++ b/internal/gateway/stats.go @@ -395,6 +395,51 @@ func (s *Stats) Records(from, to int64, key string) []Req { return out } +// AuditRecords returns every request row from the audit file plus its rotated +// .old files within the [from,to] unix-millisecond window, optionally for one +// masked key id. Unlike Records it is not bounded by the in-memory ring, so it +// yields a complete history for CSV export regardless of restarts or churn. +func (s *Stats) AuditRecords(from, to int64, key string) []Req { + s.mu.Lock() + path := s.auditPath + s.mu.Unlock() + if path == "" { + return nil + } + files := []string{path} + if olds, err := filepath.Glob(path + ".*.old"); err == nil { + files = append(files, olds...) + } + var out []Req + for _, p := range files { + f, err := os.Open(p) + if err != nil { + continue + } + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64*1024), 16*1024*1024) + for sc.Scan() { + var r Req + if json.Unmarshal(sc.Bytes(), &r) != nil || r.Type == "" { + continue + } + if key != "" && r.Key != key { + continue + } + if from > 0 && r.Time < from { + continue + } + if to > 0 && r.Time > to { + continue + } + out = append(out, r) + } + f.Close() + } + sort.Slice(out, func(i, j int) bool { return out[i].Time < out[j].Time }) + return out +} + // Snapshot returns the whole dashboard payload; when key != "" the records // and aggregate views are restricted to that gateway key. func (s *Stats) Snapshot(limit int, key string) map[string]interface{} { diff --git a/internal/gateway/ui/index.html b/internal/gateway/ui/index.html index ed5fa0d..bbfa786 100644 --- a/internal/gateway/ui/index.html +++ b/internal/gateway/ui/index.html @@ -107,11 +107,17 @@ pre.configbox { background:var(--card2); border:1px solid var(--line); border-ra .kpi .k-val { font-size:26px; font-weight:800; margin-top:4px; font-variant-numeric:tabular-nums; letter-spacing:.3px; } .kpi .k-val .u { font-size:13px; font-weight:600; color:var(--muted); margin-left:3px; } .kpi .k-sub { font-size:11.5px; color:var(--muted); margin-top:3px; } +.kpi.kpi-exit { cursor:pointer; border-color:var(--err); } +.kpi.kpi-exit:hover { border-color:var(--err); box-shadow:0 0 0 3px rgba(244,67,54,.15); } +.kpi.kpi-exit .k-lab::after { content:'×'; color:var(--err); font-weight:800; font-size:15px; margin-left:auto; } .dot { width:7px; height:7px; border-radius:50%; background:var(--ok); display:inline-block; animation:blink 1.6s infinite; } .dot.err { background:var(--err); } @keyframes blink { 50% { opacity:.25; } } .dash-row { display:flex; gap:12px; margin-bottom:18px; } -.dash-row .card { flex:1; min-width:0; } +.dash-row .card { flex:1; min-width:0; display:flex; flex-direction:column; max-height:440px; } +.dash-row .card h2 { flex:0 0 auto; } +.dash-row .card > div { flex:1 1 auto; min-height:0; overflow:auto; -webkit-overflow-scrolling:touch; } +.dash-row .card > div table thead th { position:sticky; top:0; background:var(--card); z-index:1; } td.num, th.num { text-align:right; font-variant-numeric:tabular-nums; } td .okc { color:var(--ok); font-weight:600; } td .errc { color:var(--err); font-weight:600; } @@ -482,7 +488,7 @@ const STR = { sortCooling:'冷却', sortFail:'失败', sortHealthTip:'冷却 / 失败次数 / 偏好分 实时状态', sortHealthReset:'链上冷却已复位', sortSource:'源', sortPrio:'优先级 %s', sortEmpty:'该源暂无模型', kpiActive:'活跃请求', kpiReqs:'总请求', kpiOk:'成功率', kpiTokens:'Tokens', kpiLat:'平均延迟', kpiMaxLat:'最大延迟', - dashModel:'模型用量', dashSrc:'源用量与延迟', dashKey:'密钥用量', dashRecs:'请求记录', exportCsv:'导出 CSV', expWeek:'近一周', expMonth:'近一月', expYear:'近一年', expRange:'自定义范围', expStart:'开始日期', expEnd:'结束日期', expDownload:'下载', expKeysCsv:'导出密钥用量', + dashModel:'模型用量', dashSrc:'源用量与延迟', dashKey:'密钥用量', dashRecs:'请求记录', exportCsv:'导出 CSV', expWeek:'近一周', expMonth:'近一月', expYear:'近一年', expRange:'自定义范围', expStart:'开始日期', expEnd:'结束日期', expDownload:'下载', expKeysCsv:'导出密钥用量', exitKeyView:'退出该密钥视图', dashStatus:'状态码分布', thCode:'状态码', statusTag:'状态码分类统计（含 402 欠费 / 400 schema 错误；两者不计入上游退避但单独计数）', thModel:'模型', thSrc:'源', thKey:'密钥', thReqs:'请求', thOk:'成功', thErr:'失败', thPrompt:'输入 Tokens', thCompl:'输出 Tokens', thAvgLat:'平均延迟', thMaxLat:'最长延迟', @@ -538,7 +544,7 @@ kMeTitle:'My key', kMeRole:'Role', kMeModels:'Models I can use', kMeHint:'Keys c sortCooling:'cooling', sortFail:'fail', sortHealthTip:'live cooldown / failures / preference score', sortHealthReset:'chain cooldowns reset', sortSource:'source', sortPrio:'priority %s', sortEmpty:'no models in this source', kpiActive:'Active requests', kpiReqs:'Requests', kpiOk:'Success rate', kpiTokens:'Tokens', kpiLat:'Avg latency', kpiMaxLat:'Max latency', - dashModel:'Model usage', dashSrc:'Source usage & latency', dashKey:'Key usage', dashRecs:'Request records', exportCsv:'Export CSV', expWeek:'Last week', expMonth:'Last month', expYear:'Last year', expRange:'Custom range', expStart:'Start date', expEnd:'End date', expDownload:'Download', + dashModel:'Model usage', dashSrc:'Source usage & latency', dashKey:'Key usage', dashRecs:'Request records', exportCsv:'Export CSV', expWeek:'Last week', expMonth:'Last month', expYear:'Last year', expRange:'Custom range', expStart:'Start date', expEnd:'End date', expDownload:'Download', expKeysCsv:'Export key usage', exitKeyView:'Exit this key view', dashStatus:'Status codes', thCode:'Code', statusTag:'Per-status aggregates — 402 quota / 400 schema errors are counted here but never back off the provider', thModel:'Model', thSrc:'Source', thKey:'Key', thReqs:'Requests', thOk:'OK', thErr:'Err', thPrompt:'Prompt Tokens', thCompl:'Completion Tokens', thAvgLat:'Avg latency', thMaxLat:'Max latency', @@ -672,7 +678,7 @@ async function renderStatus() {

${t('dashStatus')} ${t('statusTag')}

${t('dashKey')}

-

${t('dashRecs')}

+

${t('dashRecs')}

${t('recFilter')} @@ -690,15 +696,16 @@ async function renderStatus() { await paintStats(); statsTimerId = setInterval(paintStats, 3000); } -function renderKeySelect(keys) { +function renderKeySelect(keys, keyNames) { const sel = $('#rec-key'); if (!sel) return; const cur = sel.value; + const nm = (k) => (keyNames && keyNames[k] ? keyNames[k] + ' · ' + k : k); sel.innerHTML = `` + - keys.map(k => ``).join(''); + keys.map(k => ``).join(''); if (cur) sel.value = cur; } -function renderKeyF(v) { statsKeyF = v; paintStats(); } +function renderKeyF(v) { statsKeyF = (statsKeyF === v) ? '' : v; paintStats(); } function openExportModal() { const now = new Date(); const ago = d => { const x = new Date(now); x.setDate(x.getDate() - d); return x.toISOString().slice(0, 10); }; @@ -731,14 +738,14 @@ function downloadKeysCsv() { } async function paintStats() { try { - const q = '/api/stats?limit=500' + (statsKeyF ? '&key=' + encodeURIComponent(statsKeyF) : ''); + const q = '/api/stats?limit=2000' + (statsKeyF ? '&key=' + encodeURIComponent(statsKeyF) : ''); const st = await api(q); const tot = st.total || {}; const okr = tot.reqs ? Math.round(tot.ok * 100 / tot.reqs) : 0; const avg = fmtLat(tot.latency_sum_ms, tot.reqs); const kpi = $('#kpi-row'); if (kpi) kpi.innerHTML = ` -
${t('kpiActive')}
${st.active || 0}
${statsKeyF ? esc(statsKeyF) : ''}
+
${t('kpiActive')}
${st.active || 0}
${statsKeyF ? esc(st.key_names ? (st.key_names[statsKeyF] ? st.key_names[statsKeyF] + ' · ' : '') + statsKeyF : statsKeyF) : ''}
${t('kpiReqs')}
${fmtN(tot.reqs)}
${t('kpiOk')} ${okr}%
${t('kpiTokens')}
${fmtTok(tot.tokens)}
${t('thPrompt')} ${fmtTok(tot.prompt_tokens)} · ${t('thCompl')} ${fmtTok(tot.completion_tokens)}
@@ -748,7 +755,9 @@ async function paintStats() { paintStatusTable(st.by_status || []); paintKeyTable(st.by_key || [], st.key_names || {}); paintRecords(st.records || [], st.key_names || {}); - renderKeySelect((st.by_key || []).map(k => k.name)); + const rex = $('#rec-exit'); + if (rex) rex.innerHTML = statsKeyF ? `` : ''; + renderKeySelect((st.by_key || []).map(k => k.name), st.key_names || {}); } catch (e) { console.error('[stats]', e); } } function paintModelTable(rows) { diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 4dc4853..789095f 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -92,7 +92,7 @@ func (tn *TierNode) NextStart() int64 { // requests (rotation state resets when the chain is rebuilt, e.g. after // editing the rules — acceptable, the swap also resets cooldowns). type Chain struct { - Tiers []*TierNode // descending tier order + Tiers []*TierNode // ascending tier order (tier 1 = highest priority, tried first) } // TierErrors is the per-tier failure summary carried by ChainErr. Errors @@ -133,10 +133,10 @@ func (e *ChainErr) Error() string { return b.String() } -// BuildChain groups rules into descending tiers and resolves each slot's -// provider via prov. Rules whose provider resolves to nil are dropped (the -// source no longer serves the model). Slot order within a tier follows the -// configured rule order. +// BuildChain groups rules into ascending tiers (tier 1 = highest priority, +// tried first) and resolves each slot's provider via prov. Rules whose +// provider resolves to nil are dropped (the source no longer serves the +// model). Slot order within a tier follows the configured rule order. func BuildChain(rules []Rule, prov func(model, source string) Provider) *Chain { byTier := map[int][]*Slot{} var tiers []int @@ -157,7 +157,7 @@ func BuildChain(rules []Rule, prov func(model, source string) Provider) *Chain { Prov: p, }) } - sort.Slice(tiers, func(i, j int) bool { return tiers[i] > tiers[j] }) + sort.Slice(tiers, func(i, j int) bool { return tiers[i] < tiers[j] }) ch := &Chain{} for _, t := range tiers { tn := &TierNode{Tier: t, Slots: byTier[t]} @@ -221,12 +221,12 @@ func runTier(ctx context.Context, tn *TierNode, cands []*Slot, base int64, req * return tierResult{hard: hard} } -// chainDrive runs a request down the chain (plan 2.3): tiers descending, -// per-tier round-robin starting at the tier cursor, same-tier runs ordered by -// preference (negative prefs sink but stay reachable). Quota-exhausted and -// cooling slots are filtered up front; a fully busy tier is polled for a -// bounded time before falling through. Failures are summarized in *ChainErr -// for the caller to map to HTTP 503. +// chainDrive runs a request down the chain (plan 2.3): tiers ascending (tier +// 1, the highest priority, first), per-tier round-robin starting at the tier +// cursor, same-tier runs ordered by preference (negative prefs sink but stay +// reachable). Quota-exhausted and cooling slots are filtered up front; a +// fully busy tier is polled for a bounded time before falling through. +// Failures are summarized in *ChainErr for the caller to map to HTTP 503. func (s *Scheduler) chainDrive(ctx context.Context, chain *Chain, req *types.ChatRequest, exhausted func(*Slot) bool, stream bool) (*types.UnifiedResponse, <-chan types.UnifiedChunk, string, string, error) { if chain == nil || len(chain.Tiers) == 0 { return nil, nil, "", "", fmt.Errorf("no auto slot configured") diff --git a/internal/scheduler/scheduler_test.go b/internal/scheduler/scheduler_test.go index a53ae74..3fab878 100644 --- a/internal/scheduler/scheduler_test.go +++ b/internal/scheduler/scheduler_test.go @@ -97,12 +97,12 @@ func TestBuildChainTierOrdering(t *testing.T) { if len(ch.Tiers) != 3 { t.Fatalf("tiers = %d, want 3", len(ch.Tiers)) } - if ch.Tiers[0].Tier != 3 || ch.Tiers[1].Tier != 2 || ch.Tiers[2].Tier != 1 { - t.Fatalf("tier order = %d,%d,%d, want 3,2,1", ch.Tiers[0].Tier, ch.Tiers[1].Tier, ch.Tiers[2].Tier) + if ch.Tiers[0].Tier != 1 || ch.Tiers[1].Tier != 2 || ch.Tiers[2].Tier != 3 { + t.Fatalf("tier order = %d,%d,%d, want 1,2,3", ch.Tiers[0].Tier, ch.Tiers[1].Tier, ch.Tiers[2].Tier) } // same-tier slots keep configured order, unresolvable rule is dropped - if len(ch.Tiers[2].Slots) != 2 || ch.Tiers[2].Slots[0].Model != "a" || ch.Tiers[2].Slots[1].Model != "d" { - t.Fatalf("tier 1 slots = %+v", ch.Tiers[2].Slots) + if len(ch.Tiers[0].Slots) != 2 || ch.Tiers[0].Slots[0].Model != "a" || ch.Tiers[0].Slots[1].Model != "d" { + t.Fatalf("tier 1 slots = %+v", ch.Tiers[0].Slots) } } @@ -182,9 +182,9 @@ func TestChainAllBusyBoundedWaitThenNextTier(t *testing.T) { a.busy.Store(true) b.busy.Store(true) ch := BuildChain([]Rule{ - {Tier: 5, Model: "a", Source: "s1"}, - {Tier: 5, Model: "b", Source: "s2"}, - {Tier: 4, Model: "c", Source: "s3"}, + {Tier: 1, Model: "a", Source: "s1"}, + {Tier: 1, Model: "b", Source: "s2"}, + {Tier: 2, Model: "c", Source: "s3"}, }, bySource(a, b, c)) s := New(0) t0 := time.Now()