jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-09-22 18:07:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: record cache hit/miss per request in audit trail and WebUI

Browse Source 
- Req: add CacheHit and CacheMiss fields (carrying upstream cache
  accounting from either prompt_tokens_details.cached_tokens or legacy
  prompt_cache_hit_tokens)
- recordChatUsage (non-streaming): copy cache fields from resp.TokenUsage
- pumpStream (streaming): write lastUsage cache fields back onto rec at
  stream end, so streaming requests carry cache data too
- CSV export: add first_byte_ms, cache_hit_tokens, cache_miss_tokens
  columns alongside the existing latency/prompt/completion
- WebUI request-records table: add a Cache column showing hit% per row
  (green/amber tag with tooltip hit/miss breakdown; em-dash when the
  upstream reported no cache data)
This commit is contained in:
dev
2026-08-25 09:36:21 +08:00
parent 18c2385c61
commit 24609289e8
4 changed files with 44 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
internal/gateway/api.go
View File

@ -171,7 +171,7 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
for _, k := range g.core.ListKeys() {
names[keyID(k.Key)] = k.Name
}
_ = cw.Write([]string{"time", "key", "key_name", "type", "model", "source", "status", "ok", "prompt_tokens", "completion_tokens", "latency_ms", "error"})
_ = cw.Write([]string{"time", "key", "key_name", "type", "model", "source", "status", "ok", "prompt_tokens", "completion_tokens", "latency_ms", "first_byte_ms", "cache_hit_tokens", "cache_miss_tokens", "error"})
for _, rec := range g.stats.AuditRecords(from, to, key) {
_ = cw.Write([]string{
time.UnixMilli(rec.Time).Format(time.RFC3339),
@ -185,6 +185,9 @@ func (g *Gateway) handleStatsAPI(w http.ResponseWriter, r *http.Request) {
strconv.FormatInt(rec.Prompt, 10),
strconv.FormatInt(rec.Compl, 10),
strconv.FormatInt(rec.LatMs, 10),
strconv.FormatInt(rec.FirstByteMs, 10),
strconv.FormatInt(rec.CacheHit, 10),
strconv.FormatInt(rec.CacheMiss, 10),
rec.Err,
})
}