mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-21 01:17:59 +00:00
fix(anthropic): count cached input in prompt_tokens instead of dropping it
Anthropic and OpenAI disagree on what the prompt count means:
Anthropic: input_tokens EXCLUDES cached blocks; cache_read_input_tokens and
cache_creation_input_tokens are separate, additive, billed input.
OpenAI: prompt_tokens INCLUDES its cached_tokens subset.
anthropic.lua mapped input_tokens straight onto prompt, so a cache-heavy turn
was doubly wrong: the billed prompt was undercounted by the entire cache
portion, and cached_tokens could exceed prompt_tokens — a cache hit rate above
100% for any client that divides one by the other. cache_creation_input_tokens
was never read at all, so a cache-write turn silently lost those billed tokens.
Worse, the streaming path dropped the cache split entirely: message_delta
carries the FINAL usage and only mapped input/output, so every streamed
response reported no cache information even when the upstream sent it.
All three counts are now summed into prompt, with the read half exposed as
prompt_tokens_details.cached_tokens plus the DeepSeek-legacy hit/miss pair, via
one shared map_usage() used by transform_response, message_start and
message_delta. A reported zero stays distinguishable from "never reported": the
split is emitted whenever either cache field is present, and omitted entirely
when the upstream mentions neither (justwoker reports only input/output plus its
own cost fields, so its output is byte-identical to before). map_usage returns
nil for a countless object, preserving "no usage in this chunk means say
nothing" rather than reporting zeros.
message_start's placeholder count is still emitted: justwoker reports 160 there
and the real 6931 in message_delta, and the gateway's mergeUsage lets the later
non-zero value win.
This commit is contained in:
@ -1859,3 +1859,146 @@ func TestToolIDSanitizeResponse(t *testing.T) {
|
||||
t.Errorf("anthropic stream leaks illegal id: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAnthropicUsageCacheMapping pins the Anthropic→OpenAI usage conversion.
|
||||
// The two APIs disagree on what the prompt count means: Anthropic's
|
||||
// input_tokens EXCLUDES cached blocks (cache_read_input_tokens and
|
||||
// cache_creation_input_tokens are separate, additive, billed input), while
|
||||
// OpenAI's prompt_tokens INCLUDES its cached_tokens subset. Mapping
|
||||
// input_tokens straight onto prompt undercounted the billed prompt by the whole
|
||||
// cache portion and could report cached_tokens > prompt_tokens (a hit rate over
|
||||
// 100%); cache_creation_input_tokens was dropped entirely.
|
||||
func TestAnthropicUsageCacheMapping(t *testing.T) {
|
||||
vm := NewVM(freshAdapterDir(t))
|
||||
if err := vm.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer vm.Stop()
|
||||
|
||||
type usage struct {
|
||||
Prompt int `json:"prompt"`
|
||||
Completion int `json:"completion"`
|
||||
Total int `json:"total"`
|
||||
Details *struct {
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
} `json:"prompt_tokens_details"`
|
||||
Hit int `json:"prompt_cache_hit_tokens"`
|
||||
Miss int `json:"prompt_cache_miss_tokens"`
|
||||
}
|
||||
|
||||
// ---- non-streaming, cache read + cache write reported ----
|
||||
resp := `{"content":[{"type":"text","text":"ok"}],"stop_reason":"end_turn",
|
||||
"usage":{"input_tokens":1200,"cache_read_input_tokens":40000,
|
||||
"cache_creation_input_tokens":500,"output_tokens":80}}`
|
||||
out, err := vm.Transform("anthropic", "transform_response", resp)
|
||||
if err != nil {
|
||||
t.Fatalf("transform_response: %v", err)
|
||||
}
|
||||
// Each case decodes into a FRESH value: json.Unmarshal leaves fields absent
|
||||
// from the payload untouched, so a reused struct would carry the previous
|
||||
// case's prompt_tokens_details into a response that has none.
|
||||
decode := func(out string) usage {
|
||||
t.Helper()
|
||||
var r struct {
|
||||
TokenUsage usage `json:"token_usage"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &r); err != nil {
|
||||
t.Fatalf("unmarshal: %v (%s)", err, out)
|
||||
}
|
||||
return r.TokenUsage
|
||||
}
|
||||
decodeChunk := func(out string) (usage, string) {
|
||||
t.Helper()
|
||||
var c struct {
|
||||
Usage usage `json:"usage"`
|
||||
FinishReason string `json:"finish_reason"`
|
||||
}
|
||||
if err := json.Unmarshal([]byte(out), &c); err != nil {
|
||||
t.Fatalf("unmarshal chunk: %v (%s)", err, out)
|
||||
}
|
||||
return c.Usage, c.FinishReason
|
||||
}
|
||||
|
||||
u := decode(out)
|
||||
if u.Prompt != 1200+40000+500 {
|
||||
t.Errorf("prompt must sum fresh+cache_read+cache_creation (41700), got %d", u.Prompt)
|
||||
}
|
||||
if u.Total != u.Prompt+u.Completion {
|
||||
t.Errorf("total %d != prompt %d + completion %d", u.Total, u.Prompt, u.Completion)
|
||||
}
|
||||
if u.Details == nil || u.Details.CachedTokens != 40000 {
|
||||
t.Errorf("cached_tokens must carry cache_read (40000): %s", out)
|
||||
}
|
||||
if u.Details != nil && u.Details.CachedTokens > u.Prompt {
|
||||
t.Errorf("cached_tokens %d > prompt %d implies a hit rate above 100%%",
|
||||
u.Details.CachedTokens, u.Prompt)
|
||||
}
|
||||
if u.Hit != 40000 || u.Miss != 1200+500 {
|
||||
t.Errorf("hit/miss split wrong: hit=%d miss=%d (want 40000/1700)", u.Hit, u.Miss)
|
||||
}
|
||||
|
||||
// ---- a reported zero hit must stay distinguishable from "not reported" ----
|
||||
zero := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":100,"cache_read_input_tokens":0,"output_tokens":5}}`
|
||||
out, err = vm.Transform("anthropic", "transform_response", zero)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if decode(out).Details == nil {
|
||||
t.Errorf("a reported cache_read of 0 must still emit prompt_tokens_details: %s", out)
|
||||
}
|
||||
|
||||
// An upstream that never mentions caching must not gain a fabricated split
|
||||
// (justwoker reports only input_tokens/output_tokens plus its own cost fields).
|
||||
none := `{"content":[],"stop_reason":"end_turn","usage":{"input_tokens":6931,"output_tokens":1}}`
|
||||
out, err = vm.Transform("anthropic", "transform_response", none)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
plain := decode(out)
|
||||
if plain.Details != nil {
|
||||
t.Errorf("upstream reported no cache fields; details must be absent: %s", out)
|
||||
}
|
||||
if plain.Prompt != 6931 || plain.Total != 6932 {
|
||||
t.Errorf("plain usage mismapped: %s", out)
|
||||
}
|
||||
|
||||
// ---- streaming: message_delta carries the FINAL usage and must map cache ----
|
||||
delta := `{"type":"message_delta","delta":{"stop_reason":"end_turn"},
|
||||
"usage":{"input_tokens":1200,"cache_read_input_tokens":40000,"output_tokens":80}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", delta)
|
||||
if err != nil {
|
||||
t.Fatalf("stream delta: %v", err)
|
||||
}
|
||||
du, fin := decodeChunk(out)
|
||||
if du.Prompt != 41200 {
|
||||
t.Errorf("stream final prompt must include cache_read: got %d", du.Prompt)
|
||||
}
|
||||
if du.Details == nil || du.Details.CachedTokens != 40000 {
|
||||
t.Errorf("stream final usage dropped the cache split: %s", out)
|
||||
}
|
||||
if fin != "stop" {
|
||||
t.Errorf("finish_reason regressed: %s", out)
|
||||
}
|
||||
|
||||
// message_start's placeholder count must not suppress the later real one:
|
||||
// justwoker reports input_tokens=160 at message_start and 6931 at
|
||||
// message_delta, and the gateway's mergeUsage lets the later value win.
|
||||
start := `{"type":"message_start","message":{"usage":{"input_tokens":160,"output_tokens":1}}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", start)
|
||||
if err != nil {
|
||||
t.Fatalf("message_start: %v", err)
|
||||
}
|
||||
if su, _ := decodeChunk(out); su.Prompt != 160 {
|
||||
t.Errorf("message_start usage lost: %s", out)
|
||||
}
|
||||
|
||||
// A usage-less chunk must stay silent rather than reporting zeros.
|
||||
quiet := `{"type":"message_start","message":{}}`
|
||||
out, err = vm.Transform("anthropic", "transform_stream_chunk", quiet)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if strings.Contains(out, "usage") {
|
||||
t.Errorf("usage-less chunk must not emit a usage object: %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user