feat: add per-source TTFB and tokens/s metrics to status page

- Req: add FirstByteMs field (ms to first byte, tracked for streaming)
- Stat: add FirstByteSum for aggregation
- SourceAverages(): new method computing per-source avg TTFB and tokens/s
  from the in-memory ring (300s window)
- SourceStatus: add AvgFirstByteMs and AvgTokPerS fields
- pumpStream: record FirstByteMs after first SSE chunk sent to client
- singleChat/singleChatAuto: set FirstByteMs = LatMs (non-streaming)
- handleStatusAPI: populate the new SourceStatus fields from SourceAverages()
- WebUI source table: two new columns showing TTFB (s) and Tokens/s
This commit is contained in:
dev
2026-08-25 08:24:08 +08:00
parent 2a5c7bbbc7
commit 18c2385c61
7 changed files with 118 additions and 20 deletions

View File

@ -26,6 +26,10 @@ type Req struct {
Compl int64 `json:"completion_tokens"`
// LatMs total handling time ms
LatMs int64 `json:"latency_ms"`
// FirstByteMs time-to-first-byte for streaming (ms from request start
// to the first SSE chunk sent to the client); for non-streaming it
// equals LatMs. 0 when unmeasured (legacy records).
FirstByteMs int64 `json:"first_byte_ms,omitempty"`
OK bool `json:"ok"`
// Status http status code
Status int `json:"status"`
@ -35,14 +39,15 @@ type Req struct {
// Stat aggregates counters for one dimension row.
type Stat struct {
Reqs int64 `json:"reqs"`
OK int64 `json:"ok"`
Err int64 `json:"err"`
Tokens int64 `json:"tokens"`
Prompt int64 `json:"prompt_tokens"`
Compl int64 `json:"completion_tokens"`
LatSum int64 `json:"latency_sum_ms"`
LatMax int64 `json:"latency_max_ms"`
Reqs int64 `json:"reqs"`
OK int64 `json:"ok"`
Err int64 `json:"err"`
Tokens int64 `json:"tokens"`
Prompt int64 `json:"prompt_tokens"`
Compl int64 `json:"completion_tokens"`
LatSum int64 `json:"latency_sum_ms"`
LatMax int64 `json:"latency_max_ms"`
FirstByteSum int64 `json:"first_byte_sum_ms,omitempty"`
}
type agrRow struct {
@ -130,6 +135,9 @@ func incStatus(a *Stat, name string, r Req) {
if r.LatMs > a.LatMax {
a.LatMax = r.LatMs
}
if r.FirstByteMs > 0 {
a.FirstByteSum += r.FirstByteMs
}
}
func (s *Stats) LoadAudit(path string) {
@ -452,6 +460,65 @@ func (s *Stats) SourceRecent(windowSec int64) map[string][2]int64 {
return out
}
// SourceAvg carries per-source performance averages for the status page.
type SourceAvg struct {
// AvgFirstByteMs is the mean time-to-first-byte over successful
// requests in the window (0 when no measured samples).
AvgFirstByteMs int64 `json:"avg_first_byte_ms"`
// AvgTokPerS is the aggregate completion throughput:
// sum(completion_tokens) / sum(latency_seconds) (0 when no samples).
AvgTokPerS int64 `json:"avg_tok_per_s"`
// Samples is the number of successful requests the averages cover.
Samples int64 `json:"samples"`
}
// SourceAverages computes TTFB and tokens/s averages per source from the
// in-memory ring within the window (unix seconds). Only successful chat/
// stream rows count; image and failed rows are skipped.
func (s *Stats) SourceAverages(windowSec int64) map[string]SourceAvg {
s.mu.Lock()
defer s.mu.Unlock()
cut := time.Now().Unix() - windowSec
type acc struct {
fbSum, latSum, complSum, n int64
}
accs := map[string]*acc{}
for _, r := range s.recs {
if !r.OK || r.Source == "" || r.Time/1000 < cut {
continue
}
if r.Type != "chat" && r.Type != "stream" {
continue
}
if r.LatMs <= 0 {
continue
}
a := accs[r.Source]
if a == nil {
a = &acc{}
accs[r.Source] = a
}
a.latSum += r.LatMs
a.complSum += r.Compl
a.fbSum += r.FirstByteMs
if r.FirstByteMs > 0 {
a.n++
}
}
out := make(map[string]SourceAvg, len(accs))
for src, a := range accs {
avg := SourceAvg{Samples: a.n}
if a.n > 0 {
avg.AvgFirstByteMs = a.fbSum / a.n
}
if a.latSum > 0 && a.complSum > 0 {
avg.AvgTokPerS = a.complSum * 1000 / a.latSum
}
out[src] = avg
}
return out
}
// Snapshot returns the whole dashboard payload; when key != "" the records
// and aggregate views are restricted to that gateway key.
func (s *Stats) Snapshot(limit int, key string) map[string]interface{} {