jianf/ModelRouter
1
0
Fork 0
mirror of https://gitcode.com/JianFeeeee/ModelRouter.git synced 2026-09-20 08:57:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add per-source TTFB and tokens/s metrics to status page

Browse Source 
- Req: add FirstByteMs field (ms to first byte, tracked for streaming)
- Stat: add FirstByteSum for aggregation
- SourceAverages(): new method computing per-source avg TTFB and tokens/s
  from the in-memory ring (300s window)
- SourceStatus: add AvgFirstByteMs and AvgTokPerS fields
- pumpStream: record FirstByteMs after first SSE chunk sent to client
- singleChat/singleChatAuto: set FirstByteMs = LatMs (non-streaming)
- handleStatusAPI: populate the new SourceStatus fields from SourceAverages()
- WebUI source table: two new columns showing TTFB (s) and Tokens/s
This commit is contained in:
dev
2026-08-25 08:24:08 +08:00
parent 2a5c7bbbc7
commit 18c2385c61
7 changed files with 118 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
internal/gateway/chat.go
View File

@ -613,6 +613,9 @@ func (g *Gateway) singleChat(w http.ResponseWriter, ctx context.Context, cands [
recordChatUsage(rec, req, resp)
rec.Source = usedSrc
rec.Model = usedModel
// Non-streaming: the whole response arrives at once, so TTFB equals
// the total latency.
rec.FirstByteMs = rec.LatMs
g.writeRec(rec)
writeChatCompletion(w, resp, effective)
}
@ -668,7 +671,7 @@ func mergeUsage(prev, cur *types.TokenUsage) *types.TokenUsage {
// OpenAI-standard final usage chunk (empty choices) and [DONE]. modelName
// follows writeChatCompletion's rule (requested id for direct routes, exact
// slot model for AUTO).
func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan types.UnifiedChunk, modelName string) {
func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan types.UnifiedChunk, modelName string, t0 time.Time) {
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
@ -697,6 +700,9 @@ func (g *Gateway) pumpStream(w http.ResponseWriter, rec *Req, chunks <-chan type
}) {
return
}
// First SSE byte sent to the client: record time-to-first-byte for the
// source's status-page latency average.
rec.FirstByteMs = time.Since(t0).Milliseconds()
var lastUsage *types.TokenUsage
lastFinish := ""
for ck := range chunks {
@ -787,7 +793,7 @@ func (g *Gateway) streamChat(w http.ResponseWriter, ctx context.Context, cands [
// discarded it.
rec.Source = usedSrc
rec.Prompt = estimatePromptTokens(req)
g.pumpStream(w, rec, chunks, effective)
g.pumpStream(w, rec, chunks, effective, t0)
}
// singleChatAuto runs a non-streaming AUTO request down the chain (see
@ -811,6 +817,7 @@ func (g *Gateway) singleChatAuto(w http.ResponseWriter, ctx context.Context, cha
recordChatUsage(rec, req, resp)
rec.Source = usedSrc
rec.Model = usedModel
rec.FirstByteMs = rec.LatMs
g.writeRec(rec)
writeChatCompletion(w, resp, usedModel)
}
@ -838,7 +845,7 @@ func (g *Gateway) streamChatAuto(w http.ResponseWriter, ctx context.Context, cha
}
rec.Source = usedSrc
rec.Prompt = estimatePromptTokens(req)
g.pumpStream(w, rec, chunks, usedModel)
g.pumpStream(w, rec, chunks, usedModel, t0)
}
func (g *Gateway) handleImage(w http.ResponseWriter, r *http.Request) {