mirror of
https://gitcode.com/JianFeeeee/ModelRouter.git
synced 2026-09-21 01:17:59 +00:00
feat: add per-source TTFB and tokens/s metrics to status page
- Req: add FirstByteMs field (ms to first byte, tracked for streaming) - Stat: add FirstByteSum for aggregation - SourceAverages(): new method computing per-source avg TTFB and tokens/s from the in-memory ring (300s window) - SourceStatus: add AvgFirstByteMs and AvgTokPerS fields - pumpStream: record FirstByteMs after first SSE chunk sent to client - singleChat/singleChatAuto: set FirstByteMs = LatMs (non-streaming) - handleStatusAPI: populate the new SourceStatus fields from SourceAverages() - WebUI source table: two new columns showing TTFB (s) and Tokens/s
This commit is contained in:
@ -26,6 +26,10 @@ type Req struct {
|
||||
Compl int64 `json:"completion_tokens"`
|
||||
// LatMs total handling time ms
|
||||
LatMs int64 `json:"latency_ms"`
|
||||
// FirstByteMs time-to-first-byte for streaming (ms from request start
|
||||
// to the first SSE chunk sent to the client); for non-streaming it
|
||||
// equals LatMs. 0 when unmeasured (legacy records).
|
||||
FirstByteMs int64 `json:"first_byte_ms,omitempty"`
|
||||
OK bool `json:"ok"`
|
||||
// Status http status code
|
||||
Status int `json:"status"`
|
||||
@ -35,14 +39,15 @@ type Req struct {
|
||||
|
||||
// Stat aggregates counters for one dimension row.
|
||||
type Stat struct {
|
||||
Reqs int64 `json:"reqs"`
|
||||
OK int64 `json:"ok"`
|
||||
Err int64 `json:"err"`
|
||||
Tokens int64 `json:"tokens"`
|
||||
Prompt int64 `json:"prompt_tokens"`
|
||||
Compl int64 `json:"completion_tokens"`
|
||||
LatSum int64 `json:"latency_sum_ms"`
|
||||
LatMax int64 `json:"latency_max_ms"`
|
||||
Reqs int64 `json:"reqs"`
|
||||
OK int64 `json:"ok"`
|
||||
Err int64 `json:"err"`
|
||||
Tokens int64 `json:"tokens"`
|
||||
Prompt int64 `json:"prompt_tokens"`
|
||||
Compl int64 `json:"completion_tokens"`
|
||||
LatSum int64 `json:"latency_sum_ms"`
|
||||
LatMax int64 `json:"latency_max_ms"`
|
||||
FirstByteSum int64 `json:"first_byte_sum_ms,omitempty"`
|
||||
}
|
||||
|
||||
type agrRow struct {
|
||||
@ -130,6 +135,9 @@ func incStatus(a *Stat, name string, r Req) {
|
||||
if r.LatMs > a.LatMax {
|
||||
a.LatMax = r.LatMs
|
||||
}
|
||||
if r.FirstByteMs > 0 {
|
||||
a.FirstByteSum += r.FirstByteMs
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Stats) LoadAudit(path string) {
|
||||
@ -452,6 +460,65 @@ func (s *Stats) SourceRecent(windowSec int64) map[string][2]int64 {
|
||||
return out
|
||||
}
|
||||
|
||||
// SourceAvg carries per-source performance averages for the status page.
|
||||
type SourceAvg struct {
|
||||
// AvgFirstByteMs is the mean time-to-first-byte over successful
|
||||
// requests in the window (0 when no measured samples).
|
||||
AvgFirstByteMs int64 `json:"avg_first_byte_ms"`
|
||||
// AvgTokPerS is the aggregate completion throughput:
|
||||
// sum(completion_tokens) / sum(latency_seconds) (0 when no samples).
|
||||
AvgTokPerS int64 `json:"avg_tok_per_s"`
|
||||
// Samples is the number of successful requests the averages cover.
|
||||
Samples int64 `json:"samples"`
|
||||
}
|
||||
|
||||
// SourceAverages computes TTFB and tokens/s averages per source from the
|
||||
// in-memory ring within the window (unix seconds). Only successful chat/
|
||||
// stream rows count; image and failed rows are skipped.
|
||||
func (s *Stats) SourceAverages(windowSec int64) map[string]SourceAvg {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
cut := time.Now().Unix() - windowSec
|
||||
type acc struct {
|
||||
fbSum, latSum, complSum, n int64
|
||||
}
|
||||
accs := map[string]*acc{}
|
||||
for _, r := range s.recs {
|
||||
if !r.OK || r.Source == "" || r.Time/1000 < cut {
|
||||
continue
|
||||
}
|
||||
if r.Type != "chat" && r.Type != "stream" {
|
||||
continue
|
||||
}
|
||||
if r.LatMs <= 0 {
|
||||
continue
|
||||
}
|
||||
a := accs[r.Source]
|
||||
if a == nil {
|
||||
a = &acc{}
|
||||
accs[r.Source] = a
|
||||
}
|
||||
a.latSum += r.LatMs
|
||||
a.complSum += r.Compl
|
||||
a.fbSum += r.FirstByteMs
|
||||
if r.FirstByteMs > 0 {
|
||||
a.n++
|
||||
}
|
||||
}
|
||||
out := make(map[string]SourceAvg, len(accs))
|
||||
for src, a := range accs {
|
||||
avg := SourceAvg{Samples: a.n}
|
||||
if a.n > 0 {
|
||||
avg.AvgFirstByteMs = a.fbSum / a.n
|
||||
}
|
||||
if a.latSum > 0 && a.complSum > 0 {
|
||||
avg.AvgTokPerS = a.complSum * 1000 / a.latSum
|
||||
}
|
||||
out[src] = avg
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Snapshot returns the whole dashboard payload; when key != "" the records
|
||||
// and aggregate views are restricted to that gateway key.
|
||||
func (s *Stats) Snapshot(limit int, key string) map[string]interface{} {
|
||||
|
||||
Reference in New Issue
Block a user