mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 08:57:57 +00:00
feat(types): pass through upstream cache tokens in TokenUsage
dsh displays cache-hit %, but llmsproxy dropped every upstream's cache fields — deepseek prompt_cache_hit_tokens, OpenAI prompt_tokens_details. cached_tokens, anthropic cache_read_input_tokens, gemini cachedContentTokenCount. Changes: - TokenUsage: add PromptTokensDetails (with CachedTokens) + PromptCacheHit/Miss - MarshalJSON: emit prompt_tokens_details.cached_tokens (OpenAI v2 standard) and prompt_cache_hit/miss_tokens (DeepSeek legacy) — dsh reads the former first, falls back to the latter - mergeUsage: preserve cache fields across stream chunks - standardSSEChunk: parse the upstream raw prompt_tokens_details too - deepseek.lua: forward prompt_cache_hit/miss_tokens + create prompt_tokens_details from them - openai.lua: forward prompt_tokens_details.cached_tokens and legacy prompt_cache_hit/miss_tokens; normalize legacy hits into the standard object so dsh sees them regardless of upstream format - anthropic.lua: map cache_read_input_tokens → prompt_tokens_details - gemini.lua: map cachedContentTokenCount → prompt_tokens_details
This commit is contained in:
@ -651,6 +651,15 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
|
|||||||
out.Completion = cur.Completion
|
out.Completion = cur.Completion
|
||||||
}
|
}
|
||||||
out.Total = out.Prompt + out.Completion
|
out.Total = out.Prompt + out.Completion
|
||||||
|
if cur.PromptTokensDetails != nil && cur.PromptTokensDetails.CachedTokens > 0 {
|
||||||
|
out.PromptTokensDetails = cur.PromptTokensDetails
|
||||||
|
}
|
||||||
|
if cur.PromptCacheHit > 0 {
|
||||||
|
out.PromptCacheHit = cur.PromptCacheHit
|
||||||
|
}
|
||||||
|
if cur.PromptCacheMiss > 0 {
|
||||||
|
out.PromptCacheMiss = cur.PromptCacheMiss
|
||||||
|
}
|
||||||
return &out
|
return &out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -82,6 +82,13 @@ function adapter.transform_response(raw_body)
|
|||||||
unified.token_usage.prompt = resp.usage.input_tokens or 0
|
unified.token_usage.prompt = resp.usage.input_tokens or 0
|
||||||
unified.token_usage.completion = resp.usage.output_tokens or 0
|
unified.token_usage.completion = resp.usage.output_tokens or 0
|
||||||
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
|
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
|
||||||
|
-- Anthropic reports cache_read_input_tokens; normalize into
|
||||||
|
-- OpenAI-standard prompt_tokens_details.cached_tokens so clients
|
||||||
|
-- (dsh) see the cache hit count.
|
||||||
|
local cacheRead = resp.usage.cache_read_input_tokens or 0
|
||||||
|
if cacheRead > 0 then
|
||||||
|
unified.token_usage.prompt_tokens_details = { cached_tokens = cacheRead }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if resp.content and #resp.content > 0 then
|
if resp.content and #resp.content > 0 then
|
||||||
@ -107,6 +114,10 @@ function adapter.transform_stream_chunk(raw_chunk)
|
|||||||
local c = u.output_tokens or 0
|
local c = u.output_tokens or 0
|
||||||
if p > 0 or c > 0 then
|
if p > 0 or c > 0 then
|
||||||
uses = { prompt = p, completion = c, total = p + c }
|
uses = { prompt = p, completion = c, total = p + c }
|
||||||
|
local cacheRead = u.cache_read_input_tokens or 0
|
||||||
|
if cacheRead > 0 then
|
||||||
|
uses.prompt_tokens_details = { cached_tokens = cacheRead }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
if uses ~= nil then
|
if uses ~= nil then
|
||||||
|
|||||||
@ -49,6 +49,11 @@ function adapter.transform_response(raw_body)
|
|||||||
unified.token_usage.prompt = resp.usage.prompt_tokens or 0
|
unified.token_usage.prompt = resp.usage.prompt_tokens or 0
|
||||||
unified.token_usage.completion = resp.usage.completion_tokens or 0
|
unified.token_usage.completion = resp.usage.completion_tokens or 0
|
||||||
unified.token_usage.total = resp.usage.total_tokens or 0
|
unified.token_usage.total = resp.usage.total_tokens or 0
|
||||||
|
unified.token_usage.prompt_cache_hit_tokens = resp.usage.prompt_cache_hit_tokens or 0
|
||||||
|
unified.token_usage.prompt_cache_miss_tokens = resp.usage.prompt_cache_miss_tokens or 0
|
||||||
|
if resp.usage.prompt_tokens_details and resp.usage.prompt_tokens_details.cached_tokens then
|
||||||
|
unified.token_usage.prompt_tokens_details = { cached_tokens = resp.usage.prompt_tokens_details.cached_tokens }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if type(resp.choices) == "table" and #resp.choices > 0 then
|
if type(resp.choices) == "table" and #resp.choices > 0 then
|
||||||
@ -92,7 +97,12 @@ function adapter.transform_stream_chunk(raw_chunk)
|
|||||||
prompt = chunk.usage.prompt_tokens or chunk.usage.prompt or 0,
|
prompt = chunk.usage.prompt_tokens or chunk.usage.prompt or 0,
|
||||||
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
||||||
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
||||||
|
prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens or 0,
|
||||||
|
prompt_cache_miss_tokens = chunk.usage.prompt_cache_miss_tokens or 0,
|
||||||
}
|
}
|
||||||
|
if chunk.usage.prompt_tokens_details and chunk.usage.prompt_tokens_details.cached_tokens then
|
||||||
|
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if not chunk.choices or #chunk.choices == 0 then
|
if not chunk.choices or #chunk.choices == 0 then
|
||||||
|
|||||||
@ -100,6 +100,10 @@ function adapter.transform_stream_chunk(raw_chunk)
|
|||||||
local t = chunk.usageMetadata.totalTokenCount or 0
|
local t = chunk.usageMetadata.totalTokenCount or 0
|
||||||
if p > 0 or c > 0 or t > 0 then
|
if p > 0 or c > 0 or t > 0 then
|
||||||
uses = { prompt = p, completion = c, total = t }
|
uses = { prompt = p, completion = c, total = t }
|
||||||
|
local cacheRead = chunk.usageMetadata.cachedContentTokenCount or 0
|
||||||
|
if cacheRead > 0 then
|
||||||
|
uses.prompt_tokens_details = { cached_tokens = cacheRead }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|||||||
@ -33,6 +33,22 @@ function adapter.transform_response(raw_body)
|
|||||||
unified.token_usage.prompt = resp.usage.prompt_tokens or 0
|
unified.token_usage.prompt = resp.usage.prompt_tokens or 0
|
||||||
unified.token_usage.completion = resp.usage.completion_tokens or 0
|
unified.token_usage.completion = resp.usage.completion_tokens or 0
|
||||||
unified.token_usage.total = resp.usage.total_tokens or 0
|
unified.token_usage.total = resp.usage.total_tokens or 0
|
||||||
|
-- Cache passthrough: OpenAI v2 prompt_tokens_details.cached_tokens
|
||||||
|
-- and DeepSeek-legacy prompt_cache_hit/miss_tokens. dsh reads
|
||||||
|
-- prompt_tokens_details.cached_tokens (falls back to the legacy
|
||||||
|
-- standalone field), so both shapes reach clients.
|
||||||
|
local hit = 0
|
||||||
|
if type(resp.usage.prompt_tokens_details) == "table" and (resp.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
|
||||||
|
hit = resp.usage.prompt_tokens_details.cached_tokens
|
||||||
|
unified.token_usage.prompt_tokens_details = { cached_tokens = hit }
|
||||||
|
end
|
||||||
|
if (resp.usage.prompt_cache_hit_tokens or 0) > 0 then
|
||||||
|
unified.token_usage.prompt_cache_hit_tokens = resp.usage.prompt_cache_hit_tokens
|
||||||
|
unified.token_usage.prompt_cache_miss_tokens = resp.usage.prompt_cache_miss_tokens or 0
|
||||||
|
if hit == 0 then
|
||||||
|
unified.token_usage.prompt_tokens_details = { cached_tokens = resp.usage.prompt_cache_hit_tokens }
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if type(resp.choices) == "table" and #resp.choices > 0 then
|
if type(resp.choices) == "table" and #resp.choices > 0 then
|
||||||
@ -77,6 +93,13 @@ function adapter.transform_stream_chunk(raw_chunk)
|
|||||||
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
||||||
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
||||||
}
|
}
|
||||||
|
if type(chunk.usage.prompt_tokens_details) == "table" and (chunk.usage.prompt_tokens_details.cached_tokens or 0) > 0 then
|
||||||
|
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_tokens_details.cached_tokens }
|
||||||
|
elseif (chunk.usage.prompt_cache_hit_tokens or 0) > 0 then
|
||||||
|
uses.prompt_cache_hit_tokens = chunk.usage.prompt_cache_hit_tokens
|
||||||
|
uses.prompt_cache_miss_tokens = chunk.usage.prompt_cache_miss_tokens or 0
|
||||||
|
uses.prompt_tokens_details = { cached_tokens = chunk.usage.prompt_cache_hit_tokens }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
if not chunk.choices or #chunk.choices == 0 then
|
if not chunk.choices or #chunk.choices == 0 then
|
||||||
|
|||||||
@ -1064,6 +1064,11 @@ func standardSSEChunk(data string) string {
|
|||||||
Prompt int `json:"prompt"`
|
Prompt int `json:"prompt"`
|
||||||
Completion int `json:"completion"`
|
Completion int `json:"completion"`
|
||||||
Total int `json:"total"`
|
Total int `json:"total"`
|
||||||
|
PromptCacheHit int `json:"prompt_cache_hit_tokens"`
|
||||||
|
PromptCacheMiss int `json:"prompt_cache_miss_tokens"`
|
||||||
|
PromptTokensDetails *struct {
|
||||||
|
CachedTokens int `json:"cached_tokens"`
|
||||||
|
} `json:"prompt_tokens_details"`
|
||||||
} `json:"usage"`
|
} `json:"usage"`
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal([]byte(data), &raw); err != nil {
|
if err := json.Unmarshal([]byte(data), &raw); err != nil {
|
||||||
@ -1079,6 +1084,13 @@ func standardSSEChunk(data string) string {
|
|||||||
Prompt: pickFirst(pu.PromptTokens, pu.Prompt),
|
Prompt: pickFirst(pu.PromptTokens, pu.Prompt),
|
||||||
Completion: pickFirst(pu.CompletionTokens, pu.Completion),
|
Completion: pickFirst(pu.CompletionTokens, pu.Completion),
|
||||||
Total: pickFirst(pu.TotalTokens, pu.Total),
|
Total: pickFirst(pu.TotalTokens, pu.Total),
|
||||||
|
PromptCacheHit: pu.PromptCacheHit,
|
||||||
|
PromptCacheMiss: pu.PromptCacheMiss,
|
||||||
|
}
|
||||||
|
if pu.PromptTokensDetails != nil && pu.PromptTokensDetails.CachedTokens > 0 {
|
||||||
|
usage.PromptTokensDetails = &types.PromptTokensDetails{
|
||||||
|
CachedTokens: pu.PromptTokensDetails.CachedTokens,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
finish := ""
|
finish := ""
|
||||||
|
|||||||
@ -84,6 +84,22 @@ type TokenUsage struct {
|
|||||||
Prompt int `json:"prompt"`
|
Prompt int `json:"prompt"`
|
||||||
Completion int `json:"completion"`
|
Completion int `json:"completion"`
|
||||||
Total int `json:"total"`
|
Total int `json:"total"`
|
||||||
|
// PromptTokensDetails mirrors the OpenAI v2 usage.prompt_tokens_details
|
||||||
|
// object so cache-hit counts reported by OpenAI-compatible upstreams
|
||||||
|
// (and by adapters that normalize their own cache fields into it) pass
|
||||||
|
// through to clients that read it — dsh reads cached_tokens from here.
|
||||||
|
PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`
|
||||||
|
// PromptCacheHit / PromptCacheMiss carry the DeepSeek-legacy standalone
|
||||||
|
// fields; dsh falls back to prompt_cache_hit_tokens when
|
||||||
|
// prompt_tokens_details.cached_tokens is absent.
|
||||||
|
PromptCacheHit int `json:"prompt_cache_hit_tokens,omitempty"`
|
||||||
|
PromptCacheMiss int `json:"prompt_cache_miss_tokens,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PromptTokensDetails is the OpenAI v2 prompt_tokens_details object. Only
|
||||||
|
// CachedTokens is emitted (omitempty drops the whole object when zero).
|
||||||
|
type PromptTokensDetails struct {
|
||||||
|
CachedTokens int `json:"cached_tokens,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// MarshalJSON emits both the legacy short keys (prompt/completion/total, used
|
// MarshalJSON emits both the legacy short keys (prompt/completion/total, used
|
||||||
@ -98,6 +114,9 @@ func (t TokenUsage) MarshalJSON() ([]byte, error) {
|
|||||||
Prompt int `json:"prompt"`
|
Prompt int `json:"prompt"`
|
||||||
Completion int `json:"completion"`
|
Completion int `json:"completion"`
|
||||||
Total int `json:"total"`
|
Total int `json:"total"`
|
||||||
|
PromptTokensDetails *PromptTokensDetails `json:"prompt_tokens_details,omitempty"`
|
||||||
|
PromptCacheHitTokens int `json:"prompt_cache_hit_tokens,omitempty"`
|
||||||
|
PromptCacheMissTokens int `json:"prompt_cache_miss_tokens,omitempty"`
|
||||||
}{
|
}{
|
||||||
PromptTokens: t.Prompt,
|
PromptTokens: t.Prompt,
|
||||||
CompletionTokens: t.Completion,
|
CompletionTokens: t.Completion,
|
||||||
@ -105,6 +124,9 @@ func (t TokenUsage) MarshalJSON() ([]byte, error) {
|
|||||||
Prompt: t.Prompt,
|
Prompt: t.Prompt,
|
||||||
Completion: t.Completion,
|
Completion: t.Completion,
|
||||||
Total: t.Total,
|
Total: t.Total,
|
||||||
|
PromptTokensDetails: t.PromptTokensDetails,
|
||||||
|
PromptCacheHitTokens: t.PromptCacheHit,
|
||||||
|
PromptCacheMissTokens: t.PromptCacheMiss,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user