jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-09-19 16:39:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: surface zero cache hits — distinguish 'missed' from 'not reported'

Browse Source 
Live testing across the zen pool showed models report
prompt_tokens_details.cached_tokens even when the hit count is 0 (e.g.
nemotron-3-ultra-free returns cached_tokens:0, audio_tokens:0,
cache_write_tokens:0). The previous >0 guard dropped those objects, so a
cache-enabled upstream looked identical to one without cache support.

- types: PromptTokensDetails.CachedTokens always emitted (drop inner
  omitempty) so clients see cached_tokens:0 explicitly; dsh reads it as
  a 0% hit instead of 'no data'
- adapters (9): forward prompt_tokens_details whenever the upstream
  provides it (presence check instead of >0)
- Req: add cache_reported flag set when usage carried cache accounting;
  WebUI shows an amber 0% tag for reported-but-missed rows and keeps
  the em-dash only for sources that never report cache data
This commit is contained in:
dev
2026-08-25 10:04:44 +08:00
parent ec89daad62
commit 21ec8f59d8
13 changed files with 40 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
internal/gateway/chat.go
View File

@ -575,8 +575,14 @@ func recordChatUsage(rec *Req, req *types.ChatRequest, resp *types.UnifiedRespon
// read it back so the request record carries the hit/miss split.
if d := resp.TokenUsage.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
rec.CacheHit = int64(d.CachedTokens)
}
if resp.TokenUsage.PromptTokensDetails != nil {
// Upstream reported cache details (even a 0 hit) — tag the row so
// the UI can show 0% rather than “—”.
rec.CacheReported = true
} else if resp.TokenUsage.PromptCacheHit > 0 {
rec.CacheHit = int64(resp.TokenUsage.PromptCacheHit)
rec.CacheReported = true
}
if resp.TokenUsage.PromptCacheMiss > 0 {
rec.CacheMiss = int64(resp.TokenUsage.PromptCacheMiss)
@ -761,10 +767,14 @@ func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan type
tut = lastUsage
// Write the upstream's cache accounting back onto the request record
// so the audit trail carries the hit/miss split for streaming too.
if d := tut.PromptTokensDetails; d != nil && d.CachedTokens > 0 {
rec.CacheHit = int64(d.CachedTokens)
if d := tut.PromptTokensDetails; d != nil {
rec.CacheReported = true
if d.CachedTokens > 0 {
rec.CacheHit = int64(d.CachedTokens)
}
} else if tut.PromptCacheHit > 0 {
rec.CacheHit = int64(tut.PromptCacheHit)
rec.CacheReported = true
}
if tut.PromptCacheMiss > 0 {
rec.CacheMiss = int64(tut.PromptCacheMiss)

4
internal/gateway/stats.go
View File

@ -35,6 +35,10 @@ type Req struct {
// Both 0 = upstream gave no cache data.
CacheHit int64 `json:"cache_hit_tokens,omitempty"`
CacheMiss int64 `json:"cache_miss_tokens,omitempty"`
// CacheReported marks that the upstream usage reported cache
// accounting at all (even when the hit count is 0). The WebUI shows
// "0%" instead of "—" for such rows.
CacheReported bool `json:"cache_reported,omitempty"`
OK bool `json:"ok"`
// Status http status code
Status int `json:"status"`

5
internal/gateway/ui/index.html
View File

@ -1990,10 +1990,11 @@
// the upstream reported cache accounting, otherwise an em dash.
function cacheCell(r) {
const hit = r.cache_hit_tokens || 0;
if (!hit && !(r.cache_miss_tokens > 0)) return '<span class="muted">—</span>';
if (!hit && !(r.cache_miss_tokens > 0) && !r.cache_reported)
return '<span class="muted" title="上游未报告缓存数据">—</span>';
const prompt = r.prompt_tokens || 0;
const pct = prompt > 0 ? Math.round((hit * 100) / prompt) : 0;
return `<span class="tag ${pct >= 50 ? "tag-green" : pct > 0 ? "tag-amber" : ""}" title="命中 ${hit} / 未命中 ${r.cache_miss_tokens || 0}">${pct}%</span>`;
return `<span class="tag ${pct >= 50 ? "tag-green" : "tag-amber"}" title="命中 ${hit} / 未命中 ${r.cache_miss_tokens || 0}">${pct}%</span>`;
}
function showModelConfig(srcName, model) {
const el = $("#conncfg");

4
internal/lua/adapters/agentrouter.lua
View File

@ -53,7 +53,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -107,7 +107,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/deepseek.lua
View File

@ -51,7 +51,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.total = resp.usage.total_tokens or 0
unified.token_usage.prompt_cache_hit_tokens = resp.usage.prompt_cache_hit_tokens or 0
unified.token_usage.prompt_cache_miss_tokens = resp.usage.prompt_cache_miss_tokens or 0
if resp.usage.prompt_tokens_details and resp.usage.prompt_tokens_details.cached_tokens then
if resp.usage.prompt_tokens_details and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
unified.token_usage.prompt_tokens_details = { cached_tokens = resp.usage.prompt_tokens_details.cached_tokens }
end
end
@ -100,7 +100,7 @@ function adapter.transform_stream_chunk(raw_chunk)
prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens or 0,
prompt_cache_miss_tokens = chunk.usage.prompt_cache_miss_tokens or 0,
}
if chunk.usage.prompt_tokens_details and chunk.usage.prompt_tokens_details.cached_tokens then
if chunk.usage.prompt_tokens_details and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
end
end

4
internal/lua/adapters/github.lua
View File

@ -34,7 +34,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -86,7 +86,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/groq.lua
View File

@ -33,7 +33,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -85,7 +85,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/kimicode.lua
View File

@ -65,7 +65,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -118,7 +118,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/mistral.lua
View File

@ -33,7 +33,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -85,7 +85,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/openai.lua
View File

@ -38,7 +38,7 @@ function adapter.transform_response(raw_body)
-- prompt_tokens_details.cached_tokens (falls back to the legacy
-- standalone field), so both shapes reach clients.
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -93,7 +93,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/opencode.lua
View File

@ -101,7 +101,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -158,7 +158,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/lua/adapters/sensenova.lua
View File

@ -38,7 +38,7 @@ function adapter.transform_response(raw_body)
unified.token_usage.completion = resp.usage.completion_tokens or 0
unified.token_usage.total = resp.usage.total_tokens or 0
local hit = 0
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(resp.usage.prompt_tokens_details) == "table" and resp.usage.prompt_tokens_details.cached_tokens ~= nil then
hit = resp.usage.prompt_tokens_details.cached_tokens
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
end
@ -79,7 +79,7 @@ function adapter.transform_stream_chunk(raw_chunk)
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
total = chunk.usage.total_tokens or chunk.usage.total or 0,
}
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
if type(chunk.usage.prompt_tokens_details) == "table" and chunk.usage.prompt_tokens_details.cached_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens

4
internal/types/types.go
View File

@ -99,7 +99,9 @@ type TokenUsage struct {
// PromptTokensDetails is the OpenAI v2 prompt_tokens_details object. Only
// CachedTokens is emitted (omitempty drops the whole object when zero).
type PromptTokensDetails struct {
CachedTokens int `json:"cached_tokens,omitempty"`
// CachedTokens is always emitted (even 0) so clients can distinguish
// "upstream reports cache, this request missed" from "no cache data".
CachedTokens int `json:"cached_tokens"`
}
// MarshalJSON emits both the legacy short keys (prompt/completion/total, used