perf(gateway): load request logs on demand instead of holding them in memory

Startup RSS on this deployment was 56 MB with a 29 MB audit log and ~10 MB without one: LoadAudit() json-unmarshalled the ENTIRE file into the aggregates and kept a 10000-entry ring of raw records. Two more paths had the same shape — AuditRecords() materialized a whole export window into a []Req before sorting it, and a dashboard poll serialized the full ring so the browser could render 300 rows of it. The audit file is now the source of truth and memory only holds the live window: * LoadAudit replays only the last auditReplayBytes (4 MB) and drops the truncated first line; the ring default drops 10000 -> 500, which still covers both of its consumers (the status page's 5-minute SourceRecent / SourceAverages windows and the first screen of the records table). replayPartial is exported so the UI can say the totals cover a window rather than all time. Token-quota accounting is unaffected: it reads the modelHour buckets, not the ring (pinned by a test). * AuditPage(cursor, limit, key) pages records straight off disk, reading the newest file backwards in 64 KB chunks and returning as soon as the page is full. The cursor is "<file>:<offset>" and walks into rotated .old files; a cursor whose file rotated away reports rotated=true so the client can reset instead of silently skipping records. No state is cached between requests and the file handle is closed before responding, so "release when the user leaves the page" is guaranteed by never retaining anything. * StreamAuditRecords(from,to,key,fn) replaces the accumulate-then-sort export path; the CSV handler writes rows as they are read and flushes every 1000, and a write error (client gone) aborts the walk. Export memory is O(1) regardless of the window. AuditRecords is kept as a test-only wrapper. * Snapshot ships one screen (firstScreenRecords=100) by default; aggregates are untouched. * Audit rotation 64 MB x 10 -> 16 MB x 16: same 256 MB total budget, but a smaller newest file keeps the first reverse page cheap. New route: GET /api/stats/records?before=&limit=&key= (non-admins are pinned to their own key by exportKey). /api/status additionally reports adapter_pools for admins. Measured with production's 29 MB audit copied to the test instance: startup RSS 19.0 MB (was 56 MB); scrolling 10 pages (1000 records) +0.7 MB; exporting the full history (36441 rows / 4.4 MB CSV) +0.1 MB with no residual growth.
2026-10-03 15:44:05 +00:00 · 2026-08-30 08:05:54 +08:00
parent df9aeed5fb
commit d42c02b15d
5 changed files with 979 additions and 92 deletions
--- a/internal/gateway/gateway_test.go
+++ b/internal/gateway/gateway_test.go
@ -6,10 +6,13 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
+	"time"

 	"llmsproxy/internal/config"
 	"llmsproxy/internal/core"
@ -739,3 +742,175 @@ func TestDirectStreamFailoverAuditSource(t *testing.T) {
 		t.Fatalf("audit rec source=%q ok=%v, want source=b ok=true (actual serving source)", last.Source, last.OK)
 	}
 }
+
+// TestStatusExposesAdapterPools checks that the elastic Lua pool sizing is
+// visible to admins (and only to them) via /api/status, so the grow/shrink
+// algorithm can be observed instead of inferred.
+func TestStatusExposesAdapterPools(t *testing.T) {
+	g := newTestGateway(t)
+	rr := doReq(t, g, http.MethodGet, "/api/status", "")
+	if rr.Code != 200 {
+		t.Fatalf("status = %d, body=%s", rr.Code, rr.Body.String())
+	}
+	var body map[string]interface{}
+	if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	pools, ok := body["adapter_pools"].([]interface{})
+	if !ok {
+		t.Fatalf("adapter_pools missing or wrong type: %T", body["adapter_pools"])
+	}
+	if len(pools) == 0 {
+		t.Fatal("adapter_pools must list the loaded adapters")
+	}
+	first, _ := pools[0].(map[string]interface{})
+	for _, field := range []string{"name", "created", "idle", "in_use", "max", "grow_step", "shrink_step"} {
+		if _, ok := first[field]; !ok {
+			t.Errorf("pool entry missing %q: %v", field, first)
+		}
+	}
+	// idle gateway: nothing booted eagerly
+	if c, _ := first["created"].(float64); c != 0 {
+		t.Errorf("adapter pool booted eagerly, created=%v", first["created"])
+	}
+}
+
+// TestStatsRecordsAPIPaging exercises the on-demand records endpoint the
+// dashboard scrolls with: a bounded first page plus a cursor that walks back
+// through the audit file.
+func TestStatsRecordsAPIPaging(t *testing.T) {
+	g := newTestGateway(t)
+	for i := 0; i < 250; i++ {
+		g.stats.Record(Req{
+			Time: time.Now().UnixMilli() - int64(250-i)*1000,
+			Key:  keyID("sk-test"), Type: "chat", Model: "m", Source: "s",
+			Prompt: 1, Compl: 1, OK: true, Status: 200,
+		})
+	}
+
+	rr := doReq(t, g, http.MethodGet, "/api/stats/records?limit=50", "")
+	if rr.Code != 200 {
+		t.Fatalf("status = %d body=%s", rr.Code, rr.Body.String())
+	}
+	var page struct {
+		Records    []Req  `json:"records"`
+		NextCursor string `json:"next_cursor"`
+		HasMore    bool   `json:"has_more"`
+	}
+	if err := json.Unmarshal(rr.Body.Bytes(), &page); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if len(page.Records) != 50 {
+		t.Fatalf("first page = %d records, want 50", len(page.Records))
+	}
+	if !page.HasMore || page.NextCursor == "" {
+		t.Fatal("a long history must advertise more pages plus a cursor")
+	}
+	// newest first
+	for i := 1; i < len(page.Records); i++ {
+		if page.Records[i-1].Time < page.Records[i].Time {
+			t.Fatalf("page not newest-first at %d", i)
+		}
+	}
+
+	// follow the cursor: the next page must continue strictly older
+	oldest := page.Records[len(page.Records)-1].Time
+	rr2 := doReq(t, g, http.MethodGet,
+		"/api/stats/records?limit=50&before="+url.QueryEscape(page.NextCursor), "")
+	if rr2.Code != 200 {
+		t.Fatalf("page 2 status = %d", rr2.Code)
+	}
+	var page2 struct {
+		Records []Req `json:"records"`
+	}
+	if err := json.Unmarshal(rr2.Body.Bytes(), &page2); err != nil {
+		t.Fatalf("decode page 2: %v", err)
+	}
+	if len(page2.Records) == 0 {
+		t.Fatal("cursor page must return records")
+	}
+	if page2.Records[0].Time > oldest {
+		t.Fatalf("page 2 starts at %d, must continue below %d", page2.Records[0].Time, oldest)
+	}
+}
+
+// TestStatsSnapshotShipsOneScreen: a dashboard poll must not serialize the whole
+// ring, otherwise the "load the first screen only" contract is broken on the
+// wire even if the UI pages.
+func TestStatsSnapshotShipsOneScreen(t *testing.T) {
+	g := newTestGateway(t)
+	for i := 0; i < 400; i++ {
+		g.stats.Record(Req{
+			Time: time.Now().UnixMilli(), Key: keyID("sk-test"), Type: "chat",
+			Model: "m", Source: "s", Prompt: 1, Compl: 1, OK: true, Status: 200,
+		})
+	}
+	rr := doReq(t, g, http.MethodGet, "/api/stats", "")
+	var snap struct {
+		Records []Req `json:"records"`
+		Total   Stat  `json:"total"`
+	}
+	if err := json.Unmarshal(rr.Body.Bytes(), &snap); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	if len(snap.Records) > firstScreenRecords {
+		t.Fatalf("snapshot shipped %d records, want <= %d", len(snap.Records), firstScreenRecords)
+	}
+	// aggregates still cover every request
+	if snap.Total.Reqs != 400 {
+		t.Fatalf("total reqs = %d, want 400 (aggregates must not be paged away)", snap.Total.Reqs)
+	}
+}
+
+// TestStatsCsvExportStreams checks the export still emits every row in the
+// window now that it is written straight from the audit walk.
+func TestStatsCsvExportStreams(t *testing.T) {
+	g := newTestGateway(t)
+	base := time.Now().UnixMilli()
+	for i := 0; i < 120; i++ {
+		g.stats.Record(Req{
+			Time: base + int64(i)*1000, Key: keyID("sk-test"), Type: "chat",
+			Model: "m", Source: "s", Prompt: 2, Compl: 3, OK: true, Status: 200,
+		})
+	}
+	rr := doReq(t, g, http.MethodGet,
+		"/api/stats?export=csv&from="+strconv.FormatInt(base, 10)+
+			"&to="+strconv.FormatInt(base+120*1000, 10), "")
+	if rr.Code != 200 {
+		t.Fatalf("export status = %d", rr.Code)
+	}
+	lines := strings.Split(strings.TrimSpace(rr.Body.String()), "\n")
+	if len(lines) != 121 { // header + 120 rows
+		t.Fatalf("csv had %d lines, want 121 (header + 120 records)", len(lines))
+	}
+	if !strings.HasPrefix(lines[0], "time,key,key_name") {
+		t.Fatalf("unexpected csv header: %q", lines[0])
+	}
+}
+
+// TestStatsRecordsAPIScopedToOwnKey: a non-admin key must only page its own
+// records even when it asks for someone else's.
+func TestStatsRecordsAPIScopedToOwnKey(t *testing.T) {
+	g := newTestGateway(t)
+	mine := keyID("sk-test")
+	for i := 0; i < 5; i++ {
+		g.stats.Record(Req{Time: time.Now().UnixMilli(), Key: mine, Type: "chat", Model: "m", Source: "s", OK: true, Status: 200})
+	}
+	for i := 0; i < 5; i++ {
+		g.stats.Record(Req{Time: time.Now().UnixMilli(), Key: "othermask", Type: "chat", Model: "m", Source: "s", OK: true, Status: 200})
+	}
+	// sk-test is an admin in the test gateway, so it legitimately sees all keys;
+	// assert the explicit filter path instead.
+	rr := doReq(t, g, http.MethodGet, "/api/stats/records?limit=100&key=othermask", "")
+	var page struct {
+		Records []Req `json:"records"`
+	}
+	if err := json.Unmarshal(rr.Body.Bytes(), &page); err != nil {
+		t.Fatalf("decode: %v", err)
+	}
+	for _, r := range page.Records {
+		if r.Key != "othermask" {
+			t.Fatalf("key filter leaked %q", r.Key)
+		}
+	}
+}