mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 17:07:59 +00:00
feat: record cache hit/miss per request in audit trail and WebUI
- Req: add CacheHit and CacheMiss fields (carrying upstream cache accounting from either prompt_tokens_details.cached_tokens or legacy prompt_cache_hit_tokens) - recordChatUsage (non-streaming): copy cache fields from resp.TokenUsage - pumpStream (streaming): write lastUsage cache fields back onto rec at stream end, so streaming requests carry cache data too - CSV export: add first_byte_ms, cache_hit_tokens, cache_miss_tokens columns alongside the existing latency/prompt/completion - WebUI request-records table: add a Cache column showing hit% per row (green/amber tag with tooltip hit/miss breakdown; em-dash when the upstream reported no cache data)
This commit is contained in:
@ -171,7 +171,7 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
for _, k := range g.core.ListKeys() {
|
for _, k := range g.core.ListKeys() {
|
||||||
names[keyID(k.Key)] = k.Name
|
names[keyID(k.Key)] = k.Name
|
||||||
}
|
}
|
||||||
_ = cw.Write([]string{"time", "key", "key_name", "type", "model", "source", "status", "ok", "prompt_tokens", "completion_tokens", "latency_ms", "error"})
|
_ = cw.Write([]string{"time", "key", "key_name", "type", "model", "source", "status", "ok", "prompt_tokens", "completion_tokens", "latency_ms", "first_byte_ms", "cache_hit_tokens", "cache_miss_tokens", "error"})
|
||||||
for _, rec := range g.stats.AuditRecords(from, to, key) {
|
for _, rec := range g.stats.AuditRecords(from, to, key) {
|
||||||
_ = cw.Write([]string{
|
_ = cw.Write([]string{
|
||||||
time.UnixMilli(rec.Time).Format(time.RFC3339),
|
time.UnixMilli(rec.Time).Format(time.RFC3339),
|
||||||
@ -185,6 +185,9 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
|
|||||||
strconv.FormatInt(rec.Prompt, 10),
|
strconv.FormatInt(rec.Prompt, 10),
|
||||||
strconv.FormatInt(rec.Compl, 10),
|
strconv.FormatInt(rec.Compl, 10),
|
||||||
strconv.FormatInt(rec.LatMs, 10),
|
strconv.FormatInt(rec.LatMs, 10),
|
||||||
|
strconv.FormatInt(rec.FirstByteMs, 10),
|
||||||
|
strconv.FormatInt(rec.CacheHit, 10),
|
||||||
|
strconv.FormatInt(rec.CacheMiss, 10),
|
||||||
rec.Err,
|
rec.Err,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|||||||
@ -570,6 +570,17 @@ func recordChatUsage(rec *Req, req *types.ChatRequest, resp *types.UnifiedRespon
|
|||||||
if rec.Compl == 0 {
|
if rec.Compl == 0 {
|
||||||
rec.Compl = estimateTextTokens(resp.Content, resp.ReasoningContent, resp.ToolCalls)
|
rec.Compl = estimateTextTokens(resp.Content, resp.ReasoningContent, resp.ToolCalls)
|
||||||
}
|
}
|
||||||
|
// Cache accounting: whichever source format the adapter normalized into
|
||||||
|
// (prompt_tokens_details.cached_tokens or legacy prompt_cache_hit_tokens),
|
||||||
|
// read it back so the request record carries the hit/miss split.
|
||||||
|
if d := resp.TokenUsage.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
|
||||||
|
rec.CacheHit = int64(d.CachedTokens)
|
||||||
|
} else if resp.TokenUsage.PromptCacheHit > 0 {
|
||||||
|
rec.CacheHit = int64(resp.TokenUsage.PromptCacheHit)
|
||||||
|
}
|
||||||
|
if resp.TokenUsage.PromptCacheMiss > 0 {
|
||||||
|
rec.CacheMiss = int64(resp.TokenUsage.PromptCacheMiss)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeChatCompletion renders a unified response as an OpenAI
|
// writeChatCompletion renders a unified response as an OpenAI
|
||||||
@ -748,6 +759,16 @@ func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan type
|
|||||||
var tut *types.TokenUsage
|
var tut *types.TokenUsage
|
||||||
if lastUsage != nil {
|
if lastUsage != nil {
|
||||||
tut = lastUsage
|
tut = lastUsage
|
||||||
|
// Write the upstream's cache accounting back onto the request record
|
||||||
|
// so the audit trail carries the hit/miss split for streaming too.
|
||||||
|
if d := tut.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
|
||||||
|
rec.CacheHit = int64(d.CachedTokens)
|
||||||
|
} else if tut.PromptCacheHit > 0 {
|
||||||
|
rec.CacheHit = int64(tut.PromptCacheHit)
|
||||||
|
}
|
||||||
|
if tut.PromptCacheMiss > 0 {
|
||||||
|
rec.CacheMiss = int64(tut.PromptCacheMiss)
|
||||||
|
}
|
||||||
} else if rec.Prompt+rec.Compl > 0 {
|
} else if rec.Prompt+rec.Compl > 0 {
|
||||||
u := types.TokenUsage{
|
u := types.TokenUsage{
|
||||||
Prompt: int(rec.Prompt),
|
Prompt: int(rec.Prompt),
|
||||||
|
|||||||
@ -30,6 +30,11 @@ type Req struct {
|
|||||||
// to the first SSE chunk sent to the client); for non-streaming it
|
// to the first SSE chunk sent to the client); for non-streaming it
|
||||||
// equals LatMs. 0 when unmeasured (legacy records).
|
// equals LatMs. 0 when unmeasured (legacy records).
|
||||||
FirstByteMs int64 `json:"first_byte_ms,omitempty"`
|
FirstByteMs int64 `json:"first_byte_ms,omitempty"`
|
||||||
|
// CacheHit / CacheMiss carry the upstream prompt-cache accounting
|
||||||
|
// (DeepSeek-style hit/miss tokens) when the upstream reports it.
|
||||||
|
// Both 0 = upstream gave no cache data.
|
||||||
|
CacheHit int64 `json:"cache_hit_tokens,omitempty"`
|
||||||
|
CacheMiss int64 `json:"cache_miss_tokens,omitempty"`
|
||||||
OK bool `json:"ok"`
|
OK bool `json:"ok"`
|
||||||
// Status http status code
|
// Status http status code
|
||||||
Status int `json:"status"`
|
Status int `json:"status"`
|
||||||
|
|||||||
@ -1971,18 +1971,30 @@
|
|||||||
const slice = records.slice().reverse().slice(0, 300);
|
const slice = records.slice().reverse().slice(0, 300);
|
||||||
el.innerHTML =
|
el.innerHTML =
|
||||||
`<table><tr><th>${t("thTime")}</th><th class="num">${t("thStatus")}</th><th>${t("thKey")}</th><th>${t("thType")}</th><th>${t("thModel")}</th><th>${t("thSrc")}</th>
|
`<table><tr><th>${t("thTime")}</th><th class="num">${t("thStatus")}</th><th>${t("thKey")}</th><th>${t("thType")}</th><th>${t("thModel")}</th><th>${t("thSrc")}</th>
|
||||||
<th class="num">${t("thPrompt")}</th><th class="num">${t("thCompl")}</th><th class="num">${t("thLatMs")}</th></tr>` +
|
<th class="num">${t("thPrompt")}</th><th class="num">${t("thCompl")}</th><th class="num">${t("thCache") || "缓存"}</th><th class="num">${t("thLatMs")}</th></tr>` +
|
||||||
slice
|
slice
|
||||||
.map(
|
.map(
|
||||||
(r) => `<tr>
|
(r) => `<tr>
|
||||||
<td class="t-tag">${fmtTime(r.time)}</td>
|
<td class="t-tag">${fmtTime(r.time)}</td>
|
||||||
<td class="num">${r.ok ? `<span class="tag tag-green">${r.status || 200}</span>` : `<span class="tag tag-red" title="${esc(r.error || "")}">${r.status || 500}</span>`}</td>
|
<td class="num">${r.ok ? `<span class="tag tag-green">${r.status || 200}</span>` : `<span class="tag tag-red" title="${esc(r.error || "")}">${r.status || 500}</span>`}</td>
|
||||||
<td>${esc(keyNames[r.key] ? keyNames[r.key] + " · " + r.key : r.key)}</td><td class="t-tag">${esc(r.type)}</td><td>${esc(r.model)}</td><td>${esc(r.source || "")}</td>
|
<td>${esc(keyNames[r.key] ? keyNames[r.key] + " · " + r.key : r.key)}</td><td class="t-tag">${esc(r.type)}</td><td>${esc(r.model)}</td><td>${esc(r.source || "")}</td>
|
||||||
<td class="num">${fmtTok(r.prompt_tokens)}</td><td class="num">${fmtTok(r.completion_tokens)}</td><td class="num">${fmtMs(r.latency_ms)}</td></tr>`,
|
<td class="num">${fmtTok(r.prompt_tokens)}</td><td class="num">${fmtTok(r.completion_tokens)}</td>
|
||||||
|
<td class="num">${cacheCell(r)}</td>
|
||||||
|
<td class="num">${fmtMs(r.latency_ms)}</td></tr>`,
|
||||||
)
|
)
|
||||||
.join("") +
|
.join("") +
|
||||||
"</table>";
|
"</table>";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// cacheCell renders the per-request cache-hit column: a percentage when
|
||||||
|
// the upstream reported cache accounting, otherwise an em dash.
|
||||||
|
function cacheCell(r) {
|
||||||
|
const hit = r.cache_hit_tokens || 0;
|
||||||
|
if (!hit && !(r.cache_miss_tokens > 0)) return '<span class="muted">—</span>';
|
||||||
|
const prompt = r.prompt_tokens || 0;
|
||||||
|
const pct = prompt > 0 ? Math.round((hit * 100) / prompt) : 0;
|
||||||
|
return `<span class="tag ${pct >= 50 ? "tag-green" : pct > 0 ? "tag-amber" : ""}" title="命中 ${hit} / 未命中 ${r.cache_miss_tokens || 0}">${pct}%</span>`;
|
||||||
|
}
|
||||||
function showModelConfig(srcName, model) {
|
function showModelConfig(srcName, model) {
|
||||||
const el = $("#conncfg");
|
const el = $("#conncfg");
|
||||||
if (!el) return;
|
if (!el) return;
|
||||||
|
|||||||
Reference in New Issue
Block a user