fix(anthropic): count cached input in prompt_tokens instead of dropping it

Anthropic and OpenAI disagree on what the prompt count means:

  Anthropic: input_tokens EXCLUDES cached blocks; cache_read_input_tokens and
             cache_creation_input_tokens are separate, additive, billed input.
  OpenAI:    prompt_tokens INCLUDES its cached_tokens subset.

anthropic.lua mapped input_tokens straight onto prompt, so a cache-heavy turn
was doubly wrong: the billed prompt was undercounted by the entire cache
portion, and cached_tokens could exceed prompt_tokens — a cache hit rate above
100% for any client that divides one by the other. cache_creation_input_tokens
was never read at all, so a cache-write turn silently lost those billed tokens.

Worse, the streaming path dropped the cache split entirely: message_delta
carries the FINAL usage and only mapped input/output, so every streamed
response reported no cache information even when the upstream sent it.

All three counts are now summed into prompt, with the read half exposed as
prompt_tokens_details.cached_tokens plus the DeepSeek-legacy hit/miss pair, via
one shared map_usage() used by transform_response, message_start and
message_delta. A reported zero stays distinguishable from "never reported": the
split is emitted whenever either cache field is present, and omitted entirely
when the upstream mentions neither (justwoker reports only input/output plus its
own cost fields, so its output is byte-identical to before). map_usage returns
nil for a countless object, preserving "no usage in this chunk means say
nothing" rather than reporting zeros.

message_start's placeholder count is still emitted: justwoker reports 160 there
and the real 6931 in message_delta, and the gateway's mergeUsage lets the later
non-zero value win.
This commit is contained in:
JianFeeeee
2026-09-06 09:55:22 +08:00
parent 25eb30c15b
commit 6bdb9fcc44
2 changed files with 193 additions and 31 deletions

View File

@ -42,6 +42,42 @@ local function safe_tool_id(id)
return clean .. "_" .. digest
end
-- map_usage converts one Anthropic usage object to the gateway's TokenUsage
-- shape. The two APIs disagree on what the prompt count MEANS:
--
-- Anthropic: input_tokens EXCLUDES cached blocks. cache_read_input_tokens and
-- cache_creation_input_tokens are separate, additive, billed input.
-- OpenAI: prompt_tokens INCLUDES its cached_tokens subset.
--
-- Mapping input_tokens straight onto prompt therefore did two wrong things at
-- once: it undercounted the billed prompt by the entire cache portion, and it
-- could report cached_tokens > prompt_tokens — a cache hit rate above 100%.
-- cache_creation_input_tokens was not read at all, so a cache-write turn lost
-- those billed tokens outright. Summing all three restores OpenAI semantics,
-- which is what clients (and the gateway's own audit trail) assume.
--
-- Returns nil when the object carries no counts, so callers can keep treating
-- "no usage in this chunk" as "say nothing" rather than reporting zeros.
local function map_usage(u)
if type(u) ~= "table" then return nil end
local fresh = u.input_tokens or 0
local read = u.cache_read_input_tokens or 0
local create = u.cache_creation_input_tokens or 0
local p = fresh + read + create
local c = u.output_tokens or 0
if p <= 0 and c <= 0 then return nil end
local uses = { prompt = p, completion = c, total = p + c }
-- Emit the split whenever Anthropic reports either cache field, even at 0:
-- a reported zero-hit ("cache missed") must stay distinguishable from "this
-- upstream never reports cache info at all".
if u.cache_read_input_tokens ~= nil or u.cache_creation_input_tokens ~= nil then
uses.prompt_tokens_details = { cached_tokens = read }
uses.prompt_cache_hit_tokens = read
uses.prompt_cache_miss_tokens = fresh + create
end
return uses
end
local function collect_blocks(content)
if type(content) == "string" then
if content == "" then return {} end
@ -248,17 +284,9 @@ function adapter.transform_response(raw_body)
token_usage = { prompt = 0, completion = 0, total = 0 },
}
if resp.usage then
unified.token_usage.prompt = resp.usage.input_tokens or 0
unified.token_usage.completion = resp.usage.output_tokens or 0
unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0)
-- Emit details whenever Anthropic reports the field, even at 0, so a
-- reported cache miss stays distinguishable from "not reported".
if resp.usage.cache_read_input_tokens ~= nil then
unified.token_usage.prompt_tokens_details = {
cached_tokens = resp.usage.cache_read_input_tokens
}
end
local mapped = map_usage(resp.usage)
if mapped ~= nil then
unified.token_usage = mapped
end
if resp.content and #resp.content > 0 then
@ -303,17 +331,13 @@ function adapter.transform_stream_chunk(raw_chunk)
-- ── message_start: initial usage ─────────────────────────
if chunk.type == "message_start" then
if chunk.message and type(chunk.message.usage) == "table" then
local u = chunk.message.usage
local p = u.input_tokens or 0
local c = u.output_tokens or 0
if p > 0 or c > 0 then
local uses = { prompt = p, completion = c, total = p + c }
if u.cache_read_input_tokens ~= nil then
uses.prompt_tokens_details = {
cached_tokens = u.cache_read_input_tokens
}
end
if chunk.message then
-- Some Anthropic-compatible upstreams report a placeholder here and
-- only send the true prompt count in message_delta (justwoker:
-- 160 at message_start vs 6931 at message_delta). The gateway's
-- mergeUsage lets a later non-zero value win, so both are emitted.
local uses = map_usage(chunk.message.usage)
if uses ~= nil then
return json.encode({ usage = uses, done = false })
end
end
@ -333,15 +357,10 @@ function adapter.transform_stream_chunk(raw_chunk)
finish = "stop"
end
end
local uses = nil
if type(chunk.usage) == "table" then
local u = chunk.usage
local p = u.input_tokens or 0
local c = u.output_tokens or 0
if p > 0 or c > 0 then
uses = { prompt = p, completion = c, total = p + c }
end
end
-- message_delta carries the FINAL usage, so it must map the cache
-- fields too; the old code dropped them here, which silently lost the
-- whole cache split on every streamed response.
local uses = map_usage(chunk.usage)
if uses ~= nil then
return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish, usage = uses })
end