feat(webui): scroll-paged record table, pool metrics, probe-aware health tags

Front-end half of the on-demand log loading plus the observability for the two
new scheduling mechanisms.

Records table:
  * the first screen comes from the dashboard poll, and an IntersectionObserver
    sentinel below the last row pulls the next page from /api/stats/records as
    the user scrolls.
  * rendered rows are capped at RECS_MAX_DOM=1000 (oldest rendered rows are
    dropped) so a long scroll cannot grow the DOM without bound, with a "back
    to newest" button to reset cheaply.
  * releaseRecords() drops the buffer, disconnects the observer and aborts the
    in-flight fetch (AbortController) on tab switch, on key-filter change and
    on pagehide/beforeunload — leaving the page releases everything at once.
  * the poll no longer rebuilds the table once extra pages are loaded, so the
    5s refresh cannot throw away scrolled history.
  * when the server reports replay_partial, the filter row states that the
    aggregates cover the recent audit tail and points at CSV for full history.

Adapters page shows each pool as created/max plus in_use/idle and the live
grow/shrink steps, with the sizing rule in the hover text.

Priority page health tags now distinguish cooling / probe-ready / probing
instead of a flat "cooling", and the tooltip spells out when the window opened,
when the single probe is allowed through and when the slot clears completely.
core.AutoSlotState carries cooldown_from / probe_after / probing / probeable to
feed this.
This commit is contained in:
JianFeeeee
2026-08-30 08:06:11 +08:00
parent d42c02b15d
commit 6575058556
2 changed files with 321 additions and 38 deletions

View File

@ -565,6 +565,16 @@ type AutoSlotState struct {
FailCount int64 `json:"fail_count"`
CooldownUntil int64 `json:"cooldown_until"`
Cooling bool `json:"cooling"`
// CooldownFrom is when the current cooldown window opened and ProbeAfter is
// its midpoint: from ProbeAfter on, a single probe request is allowed
// through so a recovered upstream does not have to sit out the rest of the
// window. Probing reports whether such a probe is in flight right now.
CooldownFrom int64 `json:"cooldown_from"`
ProbeAfter int64 `json:"probe_after"`
Probing bool `json:"probing"`
// Probeable marks a cooling slot that is past its midpoint, i.e. the next
// AUTO request may use it as a probe.
Probeable bool `json:"probeable"`
}
// AutoSlotStates returns per-slot health for every slot of the current chain.
@ -582,6 +592,7 @@ func (c *Core) AutoSlotStates() []AutoSlotState {
continue
}
pref, fail, until := pp.ModelHealthInfo(sl.Model)
from, probeAfter, probing := pp.ModelProbeInfo(sl.Model)
out = append(out, AutoSlotState{
Model: sl.Model,
Source: sl.Source,
@ -589,6 +600,10 @@ func (c *Core) AutoSlotStates() []AutoSlotState {
FailCount: fail,
CooldownUntil: until,
Cooling: until > now,
CooldownFrom: from,
ProbeAfter: probeAfter,
Probing: probing,
Probeable: until > now && probeAfter > 0 && now >= probeAfter,
})
}
}

View File

@ -250,6 +250,7 @@
.bar.ok i{background:var(--ok)}
.bar.err i{background:var(--danger)}
.recs-scroll{max-height:420px;overflow:auto}
.recs-sentinel{min-height:24px;padding:6px 2px;font-size:12px;color:var(--muted);text-align:center}
.recs-scroll table th{position:sticky;top:0;background:var(--card);z-index:1}
td.t-tag{white-space:nowrap}
.tbl-wrap{overflow-x:auto;-webkit-overflow-scrolling:touch}
@ -400,6 +401,7 @@
.scr-block .scr-tag{flex:0 0 auto;font-family:'JetBrains Mono',ui-monospace,Consolas,monospace;font-size:10.5px;padding:2px 7px;border-radius:9px;background:rgba(0,0,0,.24);color:#ffe9a8;border:1px solid rgba(255,220,130,.35);cursor:pointer}
.scr-block .scr-htag{flex:0 0 auto;display:flex;gap:4px;align-items:center;font-family:'JetBrains Mono',ui-monospace,Consolas,monospace;font-size:10px;font-weight:700;cursor:help}
.scr-htag .ht-cool{padding:2px 6px;border-radius:9px;background:rgba(244,114,182,.32);color:#ffd9e5;border:1px solid rgba(255,127,172,.5)}
.scr-htag .ht-probe{padding:2px 6px;border-radius:9px;background:rgba(251,191,36,.32);color:#fff3d0;border:1px solid rgba(251,191,36,.55)}
.scr-htag .ht-fail{padding:2px 6px;border-radius:9px;background:rgba(255,160,60,.22);color:#ffd9a8;border:1px solid rgba(255,180,90,.42)}
.scr-htag .ht-pref{padding:2px 6px;border-radius:9px;background:rgba(136,192,208,.26);color:#d6f1fa;border:1px solid rgba(136,192,208,.42)}
.scr-block .scr-grip{flex:0 0 auto;display:flex;flex-direction:column;gap:2px;padding:6px 4px;margin-left:2px;border-radius:6px;cursor:grab;background:rgba(255,255,255,.22);box-shadow:inset 0 1px 2px rgba(0,0,0,.2);transition:background .12s;touch-action:none}
@ -784,6 +786,14 @@
offline: "退避 / 不可用",
adTitle: "已加载适配器",
tVersion: "版本",
adPool: "Lua 状态池",
adPoolHint:
"池按需伸缩:扩容步长由最大并发决定,缩容步长由当前连接数决定(连接为 0 时回收到常驻下限)",
adPoolIdle: "空闲",
adPoolBusy: "占用",
adPoolMax: "上限",
adPoolPeak: "峰值",
adPoolSteps: "步长 (扩/缩)",
cModel: "模型",
cMsg: "Enter 发送Shift+Enter 换行",
send: "发送",
@ -865,8 +875,12 @@
sortSaved: "排序已保存并热重载",
sortHintSave: "点击保存排序后生效",
sortCooling: "冷却",
sortProbeReady: "待探测",
sortProbing: "探测中",
sortFail: "失败",
sortHealthTip: "冷却 / 失败次数 / 偏好分 实时状态",
sortProbeTip:
"冷却窗口 %s 开始,%s 起放行 1 个探测请求,%s 完全恢复(探测成功则立即恢复)",
sortHealthReset: "链上冷却已复位",
kpiActive: "活跃请求",
kpiReqs: "总请求",
@ -908,6 +922,12 @@
thLatMs: "延迟",
allKeys: "全部密钥",
recFilter: "密钥筛选",
recsLoading: "加载更多…",
recsEnd: "已到最早记录",
recsRotated: "审计日志已轮转,已从最新记录重新加载",
recsNewest: "回到最新",
recsPartial:
"统计基于最近一段审计日志(启动时只回放尾部);完整历史请导出 CSV",
thTokens: "Tokens",
noUsage: "暂无用量记录",
},
@ -991,6 +1011,14 @@
offline: "backoff / down",
adTitle: "Loaded adapters",
tVersion: "Version",
adPool: "Lua state pool",
adPoolHint:
"Pools scale on demand: the grow step follows max concurrency, the shrink step follows the live connection count (idle pools collapse to the resident floor)",
adPoolIdle: "Idle",
adPoolBusy: "In use",
adPoolMax: "Max",
adPoolPeak: "Peak",
adPoolSteps: "Steps (grow/shrink)",
cModel: "Model",
cMsg: "Enter to send, Shift+Enter for newline",
send: "Send",
@ -1073,8 +1101,12 @@
sortSaved: "Order saved & hot-reloaded",
sortHintSave: "Click Save for it to take effect",
sortCooling: "cooling",
sortProbeReady: "probe ready",
sortProbing: "probing",
sortFail: "fail",
sortHealthTip: "live cooldown / failures / preference score",
sortProbeTip:
"cooldown opened %s, one probe request is allowed from %s, fully clear at %s (a successful probe restores it immediately)",
sortHealthReset: "chain cooldowns reset",
kpiActive: "Active requests",
kpiReqs: "Requests",
@ -1116,6 +1148,12 @@
thLatMs: "Latency",
allKeys: "All keys",
recFilter: "Key filter",
recsLoading: "Loading more…",
recsEnd: "Reached the oldest record",
recsRotated: "The audit log rotated; reloaded from the newest record",
recsNewest: "Back to newest",
recsPartial:
"Aggregates cover the recent audit tail replayed at startup; export CSV for the full history",
thTokens: "Tokens",
noUsage: "No usage data yet",
},
@ -1205,6 +1243,9 @@
});
updateBreadcrumb(lastTab);
function goTab(name) {
// leaving the dashboard releases the record buffer, its observer and any
// in-flight page fetch straight away
if (lastTab === "status" && name !== "status") releaseRecords(true);
document
.querySelectorAll(".sb-i")
.forEach((x) => x.classList.toggle("active", x.dataset.tab === name));
@ -1350,6 +1391,48 @@
};
let statsKeyF = ""; // active key filter for records ('' = all)
/* ---------- records: on-demand paging ----------
* The records table holds only what is on screen. The first screen comes
* from /api/stats (its records field), and scrolling to the bottom pulls
* the next page from /api/stats/records?before=<cursor>. Nothing is cached
* across page views: leaving the dashboard (tab switch or unload) drops the
* buffer, disconnects the observer and aborts any in-flight fetch, so both
* browser and server go back to holding nothing.
*/
const RECS_PAGE = 100; // rows fetched per scroll page
const RECS_MAX_DOM = 1000; // hard cap on rendered rows (sliding window)
const recsState = {
rows: [], // newest-first, mirrors the DOM
cursor: "", // next-page cursor
hasMore: false,
loading: false,
observer: null, // IntersectionObserver on the bottom sentinel
abort: null, // AbortController for the in-flight page fetch
keyNames: {},
};
// releaseRecords drops every record-view resource. Called on tab switch,
// on key-filter change and on unload: "release when the user leaves".
function releaseRecords(clearDom) {
if (recsState.observer) {
recsState.observer.disconnect();
recsState.observer = null;
}
if (recsState.abort) {
recsState.abort.abort();
recsState.abort = null;
}
recsState.rows = [];
recsState.cursor = "";
recsState.hasMore = false;
recsState.loading = false;
recsState.keyNames = {};
if (clearDom) {
const el = $("#tb-recs");
if (el) el.innerHTML = "";
}
}
// ---- Source status column (refreshable) ----
// The column is driven by a combination of signals so a source that is
// actually serving traffic can never be shown as "down/backoff" just
@ -1458,10 +1541,12 @@
${s.sources ? `<div class="card"><h2>${t("dashSrc")}</h2><div id="tb-src"></div></div>` : ""}
</div>
<div class="card"><h2>${t("dashKey")}<span class="grow"></span><button class="ghost small" onclick="openExportModal()">${t("exportCsv")}</button></h2><div id="tb-key"></div></div>
<div class="card"><h2><span>${t("dashRecs")}</span><span class="grow"></span><span id="rec-exit"></span><button class="ghost small" onclick="openExportModal()">${t("exportCsv")}</button></h2>
<div class="card"><h2><span>${t("dashRecs")}</span><span class="grow"></span><span id="rec-exit"></span><button class="ghost small" onclick="resetRecords()">${t("recsNewest")}</button><button class="ghost small" onclick="openExportModal()">${t("exportCsv")}</button></h2>
<div class="filter-line">
<span class="muted">${t("recFilter")}</span>
<select id="rec-key" onchange="renderKeyF(this.value)"></select>
<span class="grow"></span>
<span id="rec-partial"></span>
</div>
<div class="recs-scroll" id="tb-recs"></div>
</div>
@ -1507,6 +1592,8 @@
}
function renderKeyF(v) {
statsKeyF = statsKeyF === v ? "" : v;
// the record set changes entirely: drop the paged buffer and start over
releaseRecords(true);
paintStats();
}
function openExportModal() {
@ -1571,8 +1658,11 @@
}
async function paintStats() {
try {
// limit=0 -> the server ships one screen of records; the rest is paged
// on demand from /api/stats/records as the user scrolls.
const q =
"/api/stats?limit=2000" +
"/api/stats?limit=" +
RECS_PAGE +
(statsKeyF ? "&key=" + encodeURIComponent(statsKeyF) : "");
const st = await api(q);
const tot = st.total || {};
@ -1638,6 +1728,13 @@
paintStatusTable(st.by_status || []);
paintKeyTable(st.by_key || [], st.key_names || {});
paintRecords(st.records || [], st.key_names || {});
// aggregates built from a bounded audit tail: say so instead of
// implying the numbers are all-time totals
const rpart = $("#rec-partial");
if (rpart)
rpart.innerHTML = st.replay_partial
? `<span class="muted" title="${escAttr(t("recsPartial"))}">ⓘ ${esc(t("recsPartial"))}</span>`
: "";
const rex = $("#rec-exit");
if (rex)
rex.innerHTML = statsKeyF
@ -1981,29 +2078,141 @@
.join("") +
"</table></div>";
}
function paintRecords(records, keyNames) {
const el = $("#tb-recs");
if (!el) return;
if (!records.length) {
el.innerHTML = `<div class="muted">${t("noUsage")}</div>`;
return;
// resetRecords drops the paged history and returns to the newest screen,
// which is also how the user gets out of a long scroll cheaply.
function resetRecords() {
releaseRecords(true);
paintStats();
}
const slice = records.slice().reverse().slice(0, 300);
el.innerHTML =
`<table><tr><th>${t("thTime")}</th><th class="num">${t("thStatus")}</th><th>${t("thKey")}</th><th>${t("thType")}</th><th>${t("thModel")}</th><th>${t("thSrc")}</th>
<th class="num">${t("thPrompt")}</th><th class="num">${t("thCompl")}</th><th class="num">${t("thCache") || "缓存"}</th><th class="num">${t("thLatMs")}</th></tr>` +
slice
.map(
(r) => `<tr>
// recRowHtml renders one record row.
function recRowHtml(r, keyNames) {
return `<tr>
<td class="t-tag">${fmtTime(r.time)}</td>
<td class="num">${r.ok ? `<span class="tag tag-green">${r.status || 200}</span>` : `<span class="tag tag-red" title="${esc(r.error || "")}">${r.status || 500}</span>`}</td>
<td>${esc(keyNames[r.key] ? keyNames[r.key] + " · " + r.key : r.key)}</td><td class="t-tag">${esc(r.type)}</td><td>${esc(r.model)}</td><td>${esc(r.source || "")}</td>
<td class="num">${fmtTok(r.prompt_tokens)}</td><td class="num">${fmtTok(r.completion_tokens)}</td>
<td class="num">${cacheCell(r)}</td>
<td class="num">${fmtMs(r.latency_ms)}</td></tr>`,
)
.join("") +
"</table>";
<td class="num">${fmtMs(r.latency_ms)}</td></tr>`;
}
function recHeadHtml() {
return `<tr><th>${t("thTime")}</th><th class="num">${t("thStatus")}</th><th>${t("thKey")}</th><th>${t("thType")}</th><th>${t("thModel")}</th><th>${t("thSrc")}</th>
<th class="num">${t("thPrompt")}</th><th class="num">${t("thCompl")}</th><th class="num">${t("thCache") || "缓存"}</th><th class="num">${t("thLatMs")}</th></tr>`;
}
// paintRecords renders the FIRST SCREEN from the dashboard poll. Rows are
// newest-first. Once the user has scrolled (extra pages loaded) the poll no
// longer rebuilds the table, otherwise every refresh would throw the loaded
// history away.
function paintRecords(records, keyNames) {
const el = $("#tb-recs");
if (!el) return;
recsState.keyNames = keyNames || {};
const first = (records || []).slice().reverse(); // API sends oldest-first
if (recsState.rows.length > first.length) {
// paged view is live: only refresh the key names, keep the loaded rows
return;
}
if (!first.length) {
recsState.rows = [];
el.innerHTML = `<div class="muted">${t("noUsage")}</div>`;
return;
}
recsState.rows = first;
recsState.cursor = "";
recsState.hasMore = true; // probed on the first scroll
el.innerHTML =
`<table id="recs-tbl">${recHeadHtml()}<tbody id="recs-body">` +
first.map((r) => recRowHtml(r, recsState.keyNames)).join("") +
`</tbody></table><div id="recs-sentinel" class="recs-sentinel"></div>`;
attachRecsObserver();
}
// attachRecsObserver watches a sentinel below the last row; entering the
// viewport loads the next page. The observer is torn down by
// releaseRecords when the view goes away.
function attachRecsObserver() {
if (recsState.observer) {
recsState.observer.disconnect();
recsState.observer = null;
}
const root = $("#tb-recs");
const sentinel = $("#recs-sentinel");
if (!root || !sentinel || typeof IntersectionObserver === "undefined")
return;
recsState.observer = new IntersectionObserver(
(entries) => {
if (entries.some((e) => e.isIntersecting)) loadMoreRecords();
},
{ root, rootMargin: "120px" },
);
recsState.observer.observe(sentinel);
}
// loadMoreRecords fetches the next page and appends it. Concurrent calls
// are ignored, and a stale response (view released meanwhile) is dropped.
async function loadMoreRecords() {
if (recsState.loading || recsState.hasMore === false) return;
recsState.loading = true;
const sentinel = $("#recs-sentinel");
if (sentinel) sentinel.textContent = t("recsLoading");
const ctrl = new AbortController();
recsState.abort = ctrl;
try {
let q =
"/api/stats/records?limit=" +
RECS_PAGE +
(statsKeyF ? "&key=" + encodeURIComponent(statsKeyF) : "");
if (recsState.cursor)
q += "&before=" + encodeURIComponent(recsState.cursor);
const page = await api(q, { signal: ctrl.signal });
if (recsState.abort !== ctrl) return; // released while in flight
appendRecords(page);
} catch (e) {
if (e && e.name === "AbortError") return;
console.error("[records]", e);
recsState.hasMore = false;
} finally {
if (recsState.abort === ctrl) recsState.abort = null;
recsState.loading = false;
const s = $("#recs-sentinel");
if (s)
s.textContent = recsState.hasMore === false ? t("recsEnd") : "";
}
}
// appendRecords merges one page into the table, skipping records already on
// screen (the first page overlaps the dashboard's first screen) and
// trimming the oldest DOM rows past RECS_MAX_DOM so the table cannot grow
// without bound during a long scroll.
function appendRecords(page) {
const body = $("#recs-body");
if (!body) return;
if (page.key_names) recsState.keyNames = page.key_names;
const seen = new Set(
recsState.rows.map((r) => r.time + "|" + r.key + "|" + r.model),
);
const fresh = (page.records || []).filter(
(r) => !seen.has(r.time + "|" + r.key + "|" + r.model),
);
if (fresh.length) {
body.insertAdjacentHTML(
"beforeend",
fresh.map((r) => recRowHtml(r, recsState.keyNames)).join(""),
);
recsState.rows = recsState.rows.concat(fresh);
// sliding window: drop the oldest rendered rows (bottom of the table)
const over = recsState.rows.length - RECS_MAX_DOM;
if (over > 0) {
for (let i = 0; i < over && body.lastElementChild; i++)
body.removeChild(body.lastElementChild);
recsState.rows = recsState.rows.slice(0, RECS_MAX_DOM);
}
}
recsState.cursor = page.next_cursor || "";
recsState.hasMore = !!page.has_more && !!page.next_cursor;
if (page.rotated) toast(t("recsRotated"));
}
// cacheCell renders the per-request cache-hit column: a percentage when
@ -2979,8 +3188,19 @@
const st = sortStateMap.get(it.id + "|" + (it.src || "*"));
if (!st) return "";
const bits = [];
if (st.cooling)
bits.push(`<span class="ht-cool">${esc(t("sortCooling"))}</span>`);
if (st.cooling) {
// A cooling slot past its window midpoint still takes one probe
// request, so show that instead of a flat "cooling".
if (st.probing)
bits.push(
`<span class="ht-probe">${esc(t("sortProbing"))}</span>`,
);
else if (st.probeable)
bits.push(
`<span class="ht-probe">${esc(t("sortProbeReady"))}</span>`,
);
else bits.push(`<span class="ht-cool">${esc(t("sortCooling"))}</span>`);
}
if (st.fail_count > 0)
bits.push(
`<span class="ht-fail">${esc(t("sortFail") + "×" + st.fail_count)}</span>`,
@ -2990,7 +3210,26 @@
`<span class="ht-pref">${esc(st.pref > 0 ? "+" + st.pref : "" + st.pref)}</span>`,
);
if (!bits.length) return "";
return `<span class="scr-htag" title="${escAttr(t("sortHealthTip"))}">${bits.join("")}</span>`;
return `<span class="scr-htag" title="${escAttr(healthTip(st))}">${bits.join("")}</span>`;
}
// healthTip explains the cooldown/probe window in the tooltip: when the
// window opened, when probing starts and when it fully expires.
function healthTip(st) {
const base = t("sortHealthTip");
if (!st.cooling || !st.cooldown_until) return base;
const hms = (ts) =>
ts ? new Date(ts * 1000).toLocaleTimeString() : "—";
return (
base +
"\n" +
tFmt(
"sortProbeTip",
hms(st.cooldown_from),
hms(st.probe_after),
hms(st.cooldown_until),
)
);
}
function scrBlockHtml(it, isFirst, li, ji, extraClass) {
const c = srcColor(it.src);
@ -3641,18 +3880,23 @@
/* ---------- adapters tab ---------- */
async function renderAdapters() {
const j = await api("/api/status");
const pools = {};
(j.adapter_pools || []).forEach((p) => {
pools[p.name] = p;
});
const rows = j.adapters
.map(
(
a,
) => `<tr><td><b>${esc(a.name)}</b></td><td>${esc(a.version || "")}</td>
<td><button class="danger small" onclick="delAdapter('${escAttr(a.name)}')">${t("srcDel")}</button></td></tr>`,
)
.map((a) => {
const p = pools[a.name];
return `<tr><td><b>${esc(a.name)}</b></td><td>${esc(a.version || "")}</td>
<td>${poolCell(p)}</td>
<td><button class="danger small" onclick="delAdapter('${escAttr(a.name)}')">${t("srcDel")}</button></td></tr>`;
})
.join("");
$("#tab-adapters").innerHTML = `
<div class="card"><h2>${t("adTitle")} (${j.adapters.length})</h2>
<div class="tbl-wrap"><table><tr><th>${t("tName")}</th><th>${t("tVersion")}</th><th></th></tr>
${rows || `<tr><td colspan="3" class="empty">${t("adEmpty")}</td></tr>`}</table></div>
<div class="tbl-wrap"><table><tr><th>${t("tName")}</th><th>${t("tVersion")}</th><th>${t("adPool")}</th><th></th></tr>
${rows || `<tr><td colspan="4" class="empty">${t("adEmpty")}</td></tr>`}</table></div>
<p class="muted">${t("adPoolHint")}</p>
</div>
<div class="card"><h2>${t("uploadTitle")}</h2>
<div class="dropzone" id="dz">${t("dropHint")}</div>
@ -3664,6 +3908,26 @@
</div>`;
bindDropzone();
}
// poolCell renders one adapter's elastic Lua state pool: how many states
// exist vs the ceiling, how they split between idle and checked out, and
// the current grow/shrink steps. A pool that has never been used shows a
// dash: zero resident states is the healthy default, not an error.
function poolCell(p) {
if (!p) return '<span class="muted">—</span>';
if (!p.created)
return `<span class="muted" title="${escAttr(t("adPoolMax") + " " + p.max)}">0 / ${p.max}</span>`;
const busy = p.in_use > 0;
const title = [
`${t("adPoolIdle")} ${p.idle}`,
`${t("adPoolBusy")} ${p.in_use}`,
`${t("adPoolMax")} ${p.max}`,
`${t("adPoolPeak")} ${p.peak_in_use}`,
`${t("adPoolSteps")} +${p.grow_step} / -${p.shrink_step}`,
].join(" · ");
return `<span class="tag ${busy ? "tag-green" : "tag-amber"}" title="${escAttr(title)}">${p.created} / ${p.max}</span>
<span class="muted"> ${p.in_use}${p.idle}○ +${p.grow_step}/-${p.shrink_step}</span>`;
}
function bindDropzone() {
const dz = $("#dz"),
file = $("#adp-file"),
@ -4313,6 +4577,10 @@
}
} catch (e) {}
})();
// page unload / navigation away: release the record view immediately
window.addEventListener("pagehide", () => releaseRecords(false));
window.addEventListener("beforeunload", () => releaseRecords(false));
refresh("status");
</script>
</body>