From 6bdb9fcc44a961d11cd93d16c68a216e8b069dc5 Mon Sep 17 00:00:00 2001 From: JianFeeeee Date: Sun, 6 Sep 2026 09:55:22 +0800 Subject: [PATCH] fix(anthropic): count cached input in prompt_tokens instead of dropping it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Anthropic and OpenAI disagree on what the prompt count means: Anthropic: input_tokens EXCLUDES cached blocks; cache_read_input_tokens and cache_creation_input_tokens are separate, additive, billed input. OpenAI: prompt_tokens INCLUDES its cached_tokens subset. anthropic.lua mapped input_tokens straight onto prompt, so a cache-heavy turn was doubly wrong: the billed prompt was undercounted by the entire cache portion, and cached_tokens could exceed prompt_tokens — a cache hit rate above 100% for any client that divides one by the other. cache_creation_input_tokens was never read at all, so a cache-write turn silently lost those billed tokens. Worse, the streaming path dropped the cache split entirely: message_delta carries the FINAL usage and only mapped input/output, so every streamed response reported no cache information even when the upstream sent it. All three counts are now summed into prompt, with the read half exposed as prompt_tokens_details.cached_tokens plus the DeepSeek-legacy hit/miss pair, via one shared map_usage() used by transform_response, message_start and message_delta. A reported zero stays distinguishable from "never reported": the split is emitted whenever either cache field is present, and omitted entirely when the upstream mentions neither (justwoker reports only input/output plus its own cost fields, so its output is byte-identical to before). map_usage returns nil for a countless object, preserving "no usage in this chunk means say nothing" rather than reporting zeros. message_start's placeholder count is still emitted: justwoker reports 160 there and the real 6931 in message_delta, and the gateway's mergeUsage lets the later non-zero value win. --- internal/lua/adapters/anthropic.lua | 81 ++++++++++------ internal/lua/vm_test.go | 143 ++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 31 deletions(-) diff --git a/internal/lua/adapters/anthropic.lua b/internal/lua/adapters/anthropic.lua index 30eeed3..266e2f7 100644 --- a/internal/lua/adapters/anthropic.lua +++ b/internal/lua/adapters/anthropic.lua @@ -42,6 +42,42 @@ local function safe_tool_id(id) return clean .. "_" .. digest end +-- map_usage converts one Anthropic usage object to the gateway's TokenUsage +-- shape. The two APIs disagree on what the prompt count MEANS: +-- +-- Anthropic: input_tokens EXCLUDES cached blocks. cache_read_input_tokens and +-- cache_creation_input_tokens are separate, additive, billed input. +-- OpenAI: prompt_tokens INCLUDES its cached_tokens subset. +-- +-- Mapping input_tokens straight onto prompt therefore did two wrong things at +-- once: it undercounted the billed prompt by the entire cache portion, and it +-- could report cached_tokens > prompt_tokens — a cache hit rate above 100%. +-- cache_creation_input_tokens was not read at all, so a cache-write turn lost +-- those billed tokens outright. Summing all three restores OpenAI semantics, +-- which is what clients (and the gateway's own audit trail) assume. +-- +-- Returns nil when the object carries no counts, so callers can keep treating +-- "no usage in this chunk" as "say nothing" rather than reporting zeros. +local function map_usage(u) + if type(u) ~= "table" then return nil end + local fresh = u.input_tokens or 0 + local read = u.cache_read_input_tokens or 0 + local create = u.cache_creation_input_tokens or 0 + local p = fresh + read + create + local c = u.output_tokens or 0 + if p <= 0 and c <= 0 then return nil end + local uses = { prompt = p, completion = c, total = p + c } + -- Emit the split whenever Anthropic reports either cache field, even at 0: + -- a reported zero-hit ("cache missed") must stay distinguishable from "this + -- upstream never reports cache info at all". + if u.cache_read_input_tokens ~= nil or u.cache_creation_input_tokens ~= nil then + uses.prompt_tokens_details = { cached_tokens = read } + uses.prompt_cache_hit_tokens = read + uses.prompt_cache_miss_tokens = fresh + create + end + return uses +end + local function collect_blocks(content) if type(content) == "string" then if content == "" then return {} end @@ -248,17 +284,9 @@ function adapter.transform_response(raw_body) token_usage = { prompt = 0, completion = 0, total = 0 }, } - if resp.usage then - unified.token_usage.prompt = resp.usage.input_tokens or 0 - unified.token_usage.completion = resp.usage.output_tokens or 0 - unified.token_usage.total = (resp.usage.input_tokens or 0) + (resp.usage.output_tokens or 0) - -- Emit details whenever Anthropic reports the field, even at 0, so a - -- reported cache miss stays distinguishable from "not reported". - if resp.usage.cache_read_input_tokens ~= nil then - unified.token_usage.prompt_tokens_details = { - cached_tokens = resp.usage.cache_read_input_tokens - } - end + local mapped = map_usage(resp.usage) + if mapped ~= nil then + unified.token_usage = mapped end if resp.content and #resp.content > 0 then @@ -303,17 +331,13 @@ function adapter.transform_stream_chunk(raw_chunk) -- ── message_start: initial usage ───────────────────────── if chunk.type == "message_start" then - if chunk.message and type(chunk.message.usage) == "table" then - local u = chunk.message.usage - local p = u.input_tokens or 0 - local c = u.output_tokens or 0 - if p > 0 or c > 0 then - local uses = { prompt = p, completion = c, total = p + c } - if u.cache_read_input_tokens ~= nil then - uses.prompt_tokens_details = { - cached_tokens = u.cache_read_input_tokens - } - end + if chunk.message then + -- Some Anthropic-compatible upstreams report a placeholder here and + -- only send the true prompt count in message_delta (justwoker: + -- 160 at message_start vs 6931 at message_delta). The gateway's + -- mergeUsage lets a later non-zero value win, so both are emitted. + local uses = map_usage(chunk.message.usage) + if uses ~= nil then return json.encode({ usage = uses, done = false }) end end @@ -333,15 +357,10 @@ function adapter.transform_stream_chunk(raw_chunk) finish = "stop" end end - local uses = nil - if type(chunk.usage) == "table" then - local u = chunk.usage - local p = u.input_tokens or 0 - local c = u.output_tokens or 0 - if p > 0 or c > 0 then - uses = { prompt = p, completion = c, total = p + c } - end - end + -- message_delta carries the FINAL usage, so it must map the cache + -- fields too; the old code dropped them here, which silently lost the + -- whole cache split on every streamed response. + local uses = map_usage(chunk.usage) if uses ~= nil then return json.encode({ content = "", done = (finish ~= nil), finish_reason = finish, usage = uses }) end diff --git a/internal/lua/vm_test.go b/internal/lua/vm_test.go index 2a4f47d..3e7bcd7 100644 --- a/internal/lua/vm_test.go +++ b/internal/lua/vm_test.go @@ -1859,3 +1859,146 @@ func TestToolIDSanitizeResponse(t *testing.T) { t.Errorf("anthropic stream leaks illegal id: %s", out) } } + +// TestAnthropicUsageCacheMapping pins the Anthropic→OpenAI usage conversion. +// The two APIs disagree on what the prompt count means: Anthropic's +// input_tokens EXCLUDES cached blocks (cache_read_input_tokens and +// cache_creation_input_tokens are separate, additive, billed input), while +// OpenAI's prompt_tokens INCLUDES its cached_tokens subset. Mapping +// input_tokens straight onto prompt undercounted the billed prompt by the whole +// cache portion and could report cached_tokens > prompt_tokens (a hit rate over +// 100%); cache_creation_input_tokens was dropped entirely. +func TestAnthropicUsageCacheMapping(t *testing.T) { + vm := NewVM(freshAdapterDir(t)) + if err := vm.Start(); err != nil { + t.Fatal(err) + } + defer vm.Stop() + + type usage struct { + Prompt int `json:"prompt"` + Completion int `json:"completion"` + Total int `json:"total"` + Details *struct { + CachedTokens int `json:"cached_tokens"` + } `json:"prompt_tokens_details"` + Hit int `json:"prompt_cache_hit_tokens"` + Miss int `json:"prompt_cache_miss_tokens"` + } + + // ---- non-streaming, cache read + cache write reported ---- + resp := `{"content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn", + "usage":{"input_tokens":1200,"cache_read_input_tokens":40000, + "cache_creation_input_tokens":500,"output_tokens":80}}` + out, err := vm.Transform("anthropic", "transform_response", resp) + if err != nil { + t.Fatalf("transform_response: %v", err) + } + // Each case decodes into a FRESH value: json.Unmarshal leaves fields absent + // from the payload untouched, so a reused struct would carry the previous + // case's prompt_tokens_details into a response that has none. + decode := func(out string) usage { + t.Helper() + var r struct { + TokenUsage usage `json:"token_usage"` + } + if err := json.Unmarshal([]byte(out), &r); err != nil { + t.Fatalf("unmarshal: %v (%s)", err, out) + } + return r.TokenUsage + } + decodeChunk := func(out string) (usage, string) { + t.Helper() + var c struct { + Usage usage `json:"usage"` + FinishReason string `json:"finish_reason"` + } + if err := json.Unmarshal([]byte(out), &c); err != nil { + t.Fatalf("unmarshal chunk: %v (%s)", err, out) + } + return c.Usage, c.FinishReason + } + + u := decode(out) + if u.Prompt != 1200+40000+500 { + t.Errorf("prompt must sum fresh+cache_read+cache_creation (41700), got %d", u.Prompt) + } + if u.Total != u.Prompt+u.Completion { + t.Errorf("total %d != prompt %d + completion %d", u.Total, u.Prompt, u.Completion) + } + if u.Details == nil || u.Details.CachedTokens != 40000 { + t.Errorf("cached_tokens must carry cache_read (40000): %s", out) + } + if u.Details != nil && u.Details.CachedTokens > u.Prompt { + t.Errorf("cached_tokens %d > prompt %d implies a hit rate above 100%%", + u.Details.CachedTokens, u.Prompt) + } + if u.Hit != 40000 || u.Miss != 1200+500 { + t.Errorf("hit/miss split wrong: hit=%d miss=%d (want 40000/1700)", u.Hit, u.Miss) + } + + // ---- a reported zero hit must stay distinguishable from "not reported" ---- + zero := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":100,"cache_read_input_tokens":0,"output_tokens":5}}` + out, err = vm.Transform("anthropic", "transform_response", zero) + if err != nil { + t.Fatal(err) + } + if decode(out).Details == nil { + t.Errorf("a reported cache_read of 0 must still emit prompt_tokens_details: %s", out) + } + + // An upstream that never mentions caching must not gain a fabricated split + // (justwoker reports only input_tokens/output_tokens plus its own cost fields). + none := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":6931,"output_tokens":1}}` + out, err = vm.Transform("anthropic", "transform_response", none) + if err != nil { + t.Fatal(err) + } + plain := decode(out) + if plain.Details != nil { + t.Errorf("upstream reported no cache fields; details must be absent: %s", out) + } + if plain.Prompt != 6931 || plain.Total != 6932 { + t.Errorf("plain usage mismapped: %s", out) + } + + // ---- streaming: message_delta carries the FINAL usage and must map cache ---- + delta := `{"type":"message_delta","delta":{"stop_reason":"end_turn"}, + "usage":{"input_tokens":1200,"cache_read_input_tokens":40000,"output_tokens":80}}` + out, err = vm.Transform("anthropic", "transform_stream_chunk", delta) + if err != nil { + t.Fatalf("stream delta: %v", err) + } + du, fin := decodeChunk(out) + if du.Prompt != 41200 { + t.Errorf("stream final prompt must include cache_read: got %d", du.Prompt) + } + if du.Details == nil || du.Details.CachedTokens != 40000 { + t.Errorf("stream final usage dropped the cache split: %s", out) + } + if fin != "stop" { + t.Errorf("finish_reason regressed: %s", out) + } + + // message_start's placeholder count must not suppress the later real one: + // justwoker reports input_tokens=160 at message_start and 6931 at + // message_delta, and the gateway's mergeUsage lets the later value win. + start := `{"type":"message_start","message":{"usage":{"input_tokens":160,"output_tokens":1}}}` + out, err = vm.Transform("anthropic", "transform_stream_chunk", start) + if err != nil { + t.Fatalf("message_start: %v", err) + } + if su, _ := decodeChunk(out); su.Prompt != 160 { + t.Errorf("message_start usage lost: %s", out) + } + + // A usage-less chunk must stay silent rather than reporting zeros. + quiet := `{"type":"message_start","message":{}}` + out, err = vm.Transform("anthropic", "transform_stream_chunk", quiet) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "usage") { + t.Errorf("usage-less chunk must not emit a usage object: %s", out) + } +}