mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-20 00:48:00 +00:00
feat(gateway): pass through exact upstream token usage in streams
Streaming responses now carry the upstream's real token usage instead of gateway estimates: - UnifiedChunk gains an optional Usage field; adapters (opencode, openai) extract usage from upstream stream chunks (including the final chunk with empty choices) and pass it through. - standardSSEChunk preserves usage for passthrough adapters. - Gateway emits the exact usage in the final stream chunk when available, falling back to estimates only when the upstream provided none. Non-streaming usage was already fixed to emit OpenAI-standard keys.
This commit is contained in:
@ -67,7 +67,24 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
local ok, chunk = pcall(json.decode, raw_chunk)
|
||||
if not ok then return "" end
|
||||
|
||||
if not chunk.choices or #chunk.choices == 0 then return "" end
|
||||
-- OpenAI-style streams may attach usage to a chunk with empty choices
|
||||
-- (the final usage chunk). Keys must match Go's TokenUsage json tags
|
||||
-- (prompt/completion/total); the gateway re-emits standard *_tokens.
|
||||
local uses = nil
|
||||
if type(chunk.usage) == "table" then
|
||||
uses = {
|
||||
prompt = chunk.usage.prompt_tokens or chunk.usage.prompt or 0,
|
||||
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
||||
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
||||
}
|
||||
end
|
||||
|
||||
if not chunk.choices or #chunk.choices == 0 then
|
||||
if uses ~= nil then
|
||||
return json.encode({ usage = uses, done = (chunk.usage ~= nil) })
|
||||
end
|
||||
return ""
|
||||
end
|
||||
local delta = chunk.choices[1].delta or {}
|
||||
local fr = chunk.choices[1].finish_reason
|
||||
|
||||
@ -75,11 +92,13 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
content = delta.content or "",
|
||||
done = (fr ~= nil)
|
||||
}
|
||||
if uses ~= nil then
|
||||
unified.usage = uses
|
||||
end
|
||||
if delta.reasoning_content then
|
||||
unified.reasoning_content = delta.reasoning_content
|
||||
end
|
||||
if delta.tool_calls then
|
||||
-- pass raw streaming fragments through; OpenAI clients accumulate index+id+name+arguments
|
||||
unified.tool_calls = delta.tool_calls
|
||||
end
|
||||
return json.encode(unified)
|
||||
|
||||
@ -112,7 +112,25 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
local ok, chunk = pcall(json.decode, raw_chunk)
|
||||
if not ok then return "" end
|
||||
|
||||
if not chunk.choices or #chunk.choices == 0 then return "" end
|
||||
-- OpenAI-style streams may attach usage to a chunk with empty choices
|
||||
-- (the final usage chunk). Preserve it; the gateway emits it as the
|
||||
-- terminal usage chunk. Note: keys must match Go's TokenUsage json tags
|
||||
-- (prompt/completion/total); the gateway re-emits standard *_tokens.
|
||||
local uses = nil
|
||||
if type(chunk.usage) == "table" then
|
||||
uses = {
|
||||
prompt = chunk.usage.prompt_tokens or chunk.usage.prompt or 0,
|
||||
completion = chunk.usage.completion_tokens or chunk.usage.completion or 0,
|
||||
total = chunk.usage.total_tokens or chunk.usage.total or 0,
|
||||
}
|
||||
end
|
||||
|
||||
if not chunk.choices or #chunk.choices == 0 then
|
||||
if uses ~= nil then
|
||||
return json.encode({ usage = uses, done = (chunk.usage ~= nil) })
|
||||
end
|
||||
return ""
|
||||
end
|
||||
local delta = chunk.choices[1].delta or {}
|
||||
local fr = chunk.choices[1].finish_reason
|
||||
|
||||
@ -120,6 +138,9 @@ function adapter.transform_stream_chunk(raw_chunk)
|
||||
content = delta.content or "",
|
||||
done = (fr ~= nil)
|
||||
}
|
||||
if uses ~= nil then
|
||||
unified.usage = uses
|
||||
end
|
||||
if delta.reasoning_content then
|
||||
unified.reasoning_content = delta.reasoning_content
|
||||
end
|
||||
|
||||
Reference in New Issue
Block a user