mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 17:07:59 +00:00
fix(gateway): pass through upstream token usage in streams for all adapters
The prior usage-passthrough fix only covered openai/opencode; the same empty-choices+usage drop bug remained in the 5 sibling OpenAI-compatible adapters, and non-OpenAI providers (anthropic/gemini/ollama) never surfaced streaming usage at all. - deepseek/github/groq/kimicode/mistral: preserve usage on empty-choices chunks and attach it to normal chunks (same pattern as openai.lua) - anthropic: emit usage from message_start (prompt) and message_delta (completion); gateway merges split usage additively - gemini: read usageMetadata in the stream path - ollama: fix non-streaming key (usage -> token_usage, matches UnifiedResponse json tag) and read prompt_eval_count/eval_count; surface counts from the done stream chunk - gateway: mergeUsage combines usage across chunks (non-zero fields win, total recomputed from prompt+completion) so split usage doesn't lose the prompt half; single-chunk case (OpenAI) preserved exactly - usage-only chunks: done=false (no redundant terminal stop), matching the Go fallback standardSSEChunk
This commit is contained in:
@ -95,7 +95,27 @@ end
|
||||
function adapter.transform_stream_chunk(raw_chunk)
|
||||
local ok, chunk = pcall(json.decode, raw_chunk)
|
||||
if not ok then return "" end
|
||||
if not chunk.choices or #chunk.choices == 0 then return "" end
|
||||
|
||||
-- OpenAI-style streams may attach usage to a chunk with empty choices
|
||||
-- (the final usage chunk). Keys must match Go's TokenUsage json tags
|
||||
-- (prompt/completion/total); the gateway re-emits standard *_tokens.
|
||||
local uses = nil
|
||||
if type(chunk.usage) == "table" then
|
||||
uses = {
|
||||
prompt = chunk.usage.prompt_tokens or chunk.usage.prompt or 0,
|
||||
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
||||
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
||||
}
|
||||
end
|
||||
|
||||
if not chunk.choices or #chunk.choices == 0 then
|
||||
if uses ~= nil then
|
||||
-- usage-only chunk is not a content/finish signal; the gateway
|
||||
-- emits its own terminal stop chunk and merges this usage.
|
||||
return json.encode({ usage = uses, done = false })
|
||||
end
|
||||
return ""
|
||||
end
|
||||
local delta = chunk.choices[1].delta or {}
|
||||
local fr = chunk.choices[1].finish_reason
|
||||
|
||||
@ -103,6 +123,9 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
content = delta.content or "",
|
||||
done = (fr ~= nil)
|
||||
}
|
||||
if uses ~= nil then
|
||||
unified.usage = uses
|
||||
end
|
||||
if delta.reasoning_content then
|
||||
unified.reasoning_content = delta.reasoning_content
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user