mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 08:57:57 +00:00
feat: surface zero cache hits — distinguish 'missed' from 'not reported'
Live testing across the zen pool showed models report prompt_tokens_details.cached_tokens even when the hit count is 0 (e.g. nemotron-3-ultra-free returns cached_tokens:0, audio_tokens:0, cache_write_tokens:0). The previous >0 guard dropped those objects, so a cache-enabled upstream looked identical to one without cache support. - types: PromptTokensDetails.CachedTokens always emitted (drop inner omitempty) so clients see cached_tokens:0 explicitly; dsh reads it as a 0% hit instead of 'no data' - adapters (9): forward prompt_tokens_details whenever the upstream provides it (presence check instead of >0) - Req: add cache_reported flag set when usage carried cache accounting; WebUI shows an amber 0% tag for reported-but-missed rows and keeps the em-dash only for sources that never report cache data
This commit is contained in:
@ -575,8 +575,14 @@ func recordChatUsage(rec *Req, req *types.ChatRequest, resp *types.UnifiedRespon
|
||||
// read it back so the request record carries the hit/miss split.
|
||||
if d := resp.TokenUsage.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
|
||||
rec.CacheHit = int64(d.CachedTokens)
|
||||
}
|
||||
if resp.TokenUsage.PromptTokensDetails != nil {
|
||||
// Upstream reported cache details (even a 0 hit) — tag the row so
|
||||
// the UI can show 0% rather than “—”.
|
||||
rec.CacheReported = true
|
||||
} else if resp.TokenUsage.PromptCacheHit > 0 {
|
||||
rec.CacheHit = int64(resp.TokenUsage.PromptCacheHit)
|
||||
rec.CacheReported = true
|
||||
}
|
||||
if resp.TokenUsage.PromptCacheMiss > 0 {
|
||||
rec.CacheMiss = int64(resp.TokenUsage.PromptCacheMiss)
|
||||
@ -761,10 +767,14 @@ func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan type
|
||||
tut = lastUsage
|
||||
// Write the upstream's cache accounting back onto the request record
|
||||
// so the audit trail carries the hit/miss split for streaming too.
|
||||
if d := tut.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
|
||||
rec.CacheHit = int64(d.CachedTokens)
|
||||
if d := tut.PromptTokensDetails; d != nil {
|
||||
rec.CacheReported = true
|
||||
if d.CachedTokens > 0 {
|
||||
rec.CacheHit = int64(d.CachedTokens)
|
||||
}
|
||||
} else if tut.PromptCacheHit > 0 {
|
||||
rec.CacheHit = int64(tut.PromptCacheHit)
|
||||
rec.CacheReported = true
|
||||
}
|
||||
if tut.PromptCacheMiss > 0 {
|
||||
rec.CacheMiss = int64(tut.PromptCacheMiss)
|
||||
|
||||
Reference in New Issue
Block a user