mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 17:07:59 +00:00
fix(anthropic): count cached input in prompt_tokens instead of dropping it
Anthropic and OpenAI disagree on what the prompt count means:
Anthropic: input_tokens EXCLUDES cached blocks; cache_read_input_tokens and
cache_creation_input_tokens are separate, additive, billed input.
OpenAI: prompt_tokens INCLUDES its cached_tokens subset.
anthropic.lua mapped input_tokens straight onto prompt, so a cache-heavy turn
was doubly wrong: the billed prompt was undercounted by the entire cache
portion, and cached_tokens could exceed prompt_tokens — a cache hit rate above
100% for any client that divides one by the other. cache_creation_input_tokens
was never read at all, so a cache-write turn silently lost those billed tokens.
Worse, the streaming path dropped the cache split entirely: message_delta
carries the FINAL usage and only mapped input/output, so every streamed
response reported no cache information even when the upstream sent it.
All three counts are now summed into prompt, with the read half exposed as
prompt_tokens_details.cached_tokens plus the DeepSeek-legacy hit/miss pair, via
one shared map_usage() used by transform_response, message_start and
message_delta. A reported zero stays distinguishable from "never reported": the
split is emitted whenever either cache field is present, and omitted entirely
when the upstream mentions neither (justwoker reports only input/output plus its
own cost fields, so its output is byte-identical to before). map_usage returns
nil for a countless object, preserving "no usage in this chunk means say
nothing" rather than reporting zeros.
message_start's placeholder count is still emitted: justwoker reports 160 there
and the real 6931 in message_delta, and the gateway's mergeUsage lets the later
non-zero value win.
This commit is contained in:
@ -42,6 +42,42 @@ local function safe_tool_id(id)
|
||||
return clean .. "_" .. digest
|
||||
end
|
||||
|
||||
-- map_usage converts one Anthropic usage object to the gateway's TokenUsage
|
||||
-- shape. The two APIs disagree on what the prompt count MEANS:
|
||||
--
|
||||
-- Anthropic: input_tokens EXCLUDES cached blocks. cache_read_input_tokens and
|
||||
-- cache_creation_input_tokens are separate, additive, billed input.
|
||||
-- OpenAI: prompt_tokens INCLUDES its cached_tokens subset.
|
||||
--
|
||||
-- Mapping input_tokens straight onto prompt therefore did two wrong things at
|
||||
-- once: it undercounted the billed prompt by the entire cache portion, and it
|
||||
-- could report cached_tokens > prompt_tokens — a cache hit rate above 100%.
|
||||
-- cache_creation_input_tokens was not read at all, so a cache-write turn lost
|
||||
-- those billed tokens outright. Summing all three restores OpenAI semantics,
|
||||
-- which is what clients (and the gateway's own audit trail) assume.
|
||||
--
|
||||
-- Returns nil when the object carries no counts, so callers can keep treating
|
||||
-- "no usage in this chunk" as "say nothing" rather than reporting zeros.
|
||||
local function map_usage(u)
|
||||
if type(u) ~= "table" then return nil end
|
||||
local fresh = u.input_tokens or 0
|
||||
local read = u.cache_read_input_tokens or 0
|
||||
local create = u.cache_creation_input_tokens or 0
|
||||
local p = fresh + read + create
|
||||
local c = u.output_tokens or 0
|
||||
if p <= 0 and c <= 0 then return nil end
|
||||
local uses = { prompt = p, completion = c, total = p + c }
|
||||
-- Emit the split whenever Anthropic reports either cache field, even at 0:
|
||||
-- a reported zero-hit ("cache missed") must stay distinguishable from "this
|
||||
-- upstream never reports cache info at all".
|
||||
if u.cache_read_input_tokens ~= nil or u.cache_creation_input_tokens ~= nil then
|
||||
uses.prompt_tokens_details = { cached_tokens = read }
|
||||
uses.prompt_cache_hit_tokens = read
|
||||
uses.prompt_cache_miss_tokens = fresh + create
|
||||
end
|
||||
return uses
|
||||
end
|
||||
|
||||
local function collect_blocks(content)
|
||||
if type(content) == "string" then
|
||||
if content == "" then return {} end
|
||||
@ -248,17 +284,9 @@ function adapter.transform_response(raw_body)
|
||||
token_usage = { prompt = 0, completion = 0, total = 0 },
|
||||
}
|
||||
|
||||
if resp.usage then
|
||||
unified.token_usage.prompt = resp.usage.input_tokens or 0
|
||||
unified.token_usage.completion = resp.usage.output_tokens or 0
|
||||
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
|
||||
-- Emit details whenever Anthropic reports the field, even at 0, so a
|
||||
-- reported cache miss stays distinguishable from "not reported".
|
||||
if resp.usage.cache_read_input_tokens ~= nil then
|
||||
unified.token_usage.prompt_tokens_details = {
|
||||
cached_tokens = resp.usage.cache_read_input_tokens
|
||||
}
|
||||
end
|
||||
local mapped = map_usage(resp.usage)
|
||||
if mapped ~= nil then
|
||||
unified.token_usage = mapped
|
||||
end
|
||||
|
||||
if resp.content and #resp.content > 0 then
|
||||
@ -303,17 +331,13 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
|
||||
-- ── message_start: initial usage ─────────────────────────
|
||||
if chunk.type == "message_start" then
|
||||
if chunk.message and type(chunk.message.usage) == "table" then
|
||||
local u = chunk.message.usage
|
||||
local p = u.input_tokens or 0
|
||||
local c = u.output_tokens or 0
|
||||
if p > 0 or c > 0 then
|
||||
local uses = { prompt = p, completion = c, total = p + c }
|
||||
if u.cache_read_input_tokens ~= nil then
|
||||
uses.prompt_tokens_details = {
|
||||
cached_tokens = u.cache_read_input_tokens
|
||||
}
|
||||
end
|
||||
if chunk.message then
|
||||
-- Some Anthropic-compatible upstreams report a placeholder here and
|
||||
-- only send the true prompt count in message_delta (justwoker:
|
||||
-- 160 at message_start vs 6931 at message_delta). The gateway's
|
||||
-- mergeUsage lets a later non-zero value win, so both are emitted.
|
||||
local uses = map_usage(chunk.message.usage)
|
||||
if uses ~= nil then
|
||||
return json.encode({ usage = uses, done = false })
|
||||
end
|
||||
end
|
||||
@ -333,15 +357,10 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
finish = "stop"
|
||||
end
|
||||
end
|
||||
local uses = nil
|
||||
if type(chunk.usage) == "table" then
|
||||
local u = chunk.usage
|
||||
local p = u.input_tokens or 0
|
||||
local c = u.output_tokens or 0
|
||||
if p > 0 or c > 0 then
|
||||
uses = { prompt = p, completion = c, total = p + c }
|
||||
end
|
||||
end
|
||||
-- message_delta carries the FINAL usage, so it must map the cache
|
||||
-- fields too; the old code dropped them here, which silently lost the
|
||||
-- whole cache split on every streamed response.
|
||||
local uses = map_usage(chunk.usage)
|
||||
if uses ~= nil then
|
||||
return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish, usage = uses })
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user