package gateway import ( "fmt" "os" "path/filepath" "strings" "testing" "time" ) func TestStatsByStatus(t *testing.T) { s := NewStats(100) s.Record(Req{Key: "k", Model: "m", Source: "s", Status: 200, OK: true}) s.Record(Req{Key: "k", Model: "m", Source: "s", Status: 402, OK: false}) s.Record(Req{Key: "k", Model: "m", Source: "s", Status: 400, OK: false}) snap := s.Snapshot(0, "") bs, ok := snap["by_status"].([]agrRow) if !ok { t.Fatalf("by_status missing: %#v", snap["by_status"]) } if len(bs) != 3 { t.Fatalf("want 3 status buckets, got %d: %#v", len(bs), bs) } if bs[0].Name != "200" || bs[0].OK != 1 || bs[0].Err != 0 { t.Fatalf("bucket 200 wrong: %#v", bs[0]) } if bs[1].Name != "400" || bs[1].Err != 1 { t.Fatalf("bucket 400 wrong: %#v", bs[1]) } if bs[2].Name != "402" || bs[2].Err != 1 { t.Fatalf("bucket 402 wrong: %#v", bs[2]) } } func TestAuditRotation(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") s := NewStats(10) s.LoadAudit(path) oldRotate, oldKeep := auditRotateBytes, auditKeepOld auditRotateBytes, auditKeepOld = 64, 10 defer func() { auditRotateBytes, auditKeepOld = oldRotate, oldKeep }() oldFiles := func() []string { matches, _ := filepath.Glob(path + ".*.old") return matches } for i := 0; i < 3; i++ { s.AppendAudit("ev", map[string]interface{}{"i": i}) } if got := len(oldFiles()); got != 1 { t.Fatalf("want 1 rotated file after first overflow, got %d", got) } if b, err := os.ReadFile(path); err != nil || len(b) == 0 { t.Fatalf("active audit file must continue appending: %v %d bytes", err, len(b)) } // seed 12 fake old files; the next rotation must prune back to keep=10 for i := 1; i <= 12; i++ { name := fmt.Sprintf("%s.%010d.old", path, i) _ = os.WriteFile(name, []byte("x\n"), 0644) } s.AppendAudit("ev", map[string]interface{}{"i": 98}) s.AppendAudit("ev", map[string]interface{}{"i": 99}) if got := len(oldFiles()); got != auditKeepOld { t.Fatalf("want keeper %d old files, got %d", auditKeepOld, got) } } func TestAuditRotationRecords(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") s := NewStats(10) s.LoadAudit(path) oldRotate := auditRotateBytes auditRotateBytes = 64 defer func() { auditRotateBytes = oldRotate }() for i := 0; i < 5; i++ { s.Record(Req{Key: "k", Model: "m", Source: "s", Status: 200, OK: true}) } matches, _ := filepath.Glob(path + ".*.old") if len(matches) != 1 { t.Fatalf("Record must rotate too: got %d old files", len(matches)) } } func TestLoadAuditFullReplay(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") // timestamps anchored to whole unix hours (see aggregateLocked): window // assertions must hold no matter which minute-of-hour the suite runs at, // so rows are placed relative to hour boundaries, not raw offsets from now. now := time.Now() thisHour := now.Truncate(time.Hour) twoBucketsAgo := thisHour.Add(-90 * time.Minute) // bucket H-2 prevBucketMid := thisHour.Add(-30 * time.Minute) // bucket H-1 lines := []string{ `{"obj":"access","time":1699999999000,"key":"k","method":"GET","path":"/api/stats","status":200}`, fmt.Sprintf(`{"time":%d,"key":"k","type":"chat","model":"m","source":"s","prompt_tokens":100,"completion_tokens":50,"latency_ms":10,"ok":true,"status":200}`, twoBucketsAgo.UnixMilli()), `{this is not valid json`, fmt.Sprintf(`{"time":%d,"key":"k","type":"stream","model":"m","source":"s","prompt_tokens":200,"completion_tokens":20,"latency_ms":20,"ok":false,"status":503}`, prevBucketMid.UnixMilli()), "garbage-not-json\n", fmt.Sprintf(`{"time":%d,"key":"k","type":"chat","model":"m2","source":"s","prompt_tokens":7,"completion_tokens":3,"latency_ms":5,"ok":true,"status":200}`, prevBucketMid.UnixMilli()), } loaded := strings.Join(lines, "\n") + "\n" + strings.Repeat("x", 1<<18) + "\n" // oversized row at the END proves the scanner tolerates >64KB lines and // still finishes the replay instead of truncating silently. loaded += fmt.Sprintf(`{"time":%d,"key":"k","type":"chat","model":"m","source":"s","prompt_tokens":1,"completion_tokens":1,"latency_ms":1,"ok":true,"status":200}`, now.UnixMilli()) + "\n" if err := os.WriteFile(path, []byte(loaded), 0644); err != nil { t.Fatal(err) } s := NewStats(1000) s.LoadAudit(path) got := s.byModel["m"] if got == nil || got.Tokens != 100+50+200+20+1+1 { t.Fatalf("aggregates must replay EVERY request row, got %#v", got) } if s.byModel["m2"] == nil || s.byModel["m2"].Tokens != 10 { t.Fatalf("m2 must be replayed too, got %#v", s.byModel["m2"]) } // access rows are not requests: 4 real rows, junk skipped if len(s.recs) != 4 { t.Fatalf("ring must hold only real requests, got %d rows: %#v", len(s.recs), s.recs) } // quota window rebuilt from full history. All-time and multi-hour windows // must see everything. A 1h window must NOT return all records (the old // ms/seconds unit bug made any sec>0 window return everything): buckets // are whole unix hours, so only rows in the current hour bucket qualify // — exactly the just-now row (2 tokens). if w := s.WindowTokens("m", "s", 0); w != 372 { t.Fatalf("all-time window want 372, got %d", w) } if w := s.WindowTokens("m", "s", 3*hourSec); w != 372 { t.Fatalf("3h window want 372, got %d", w) } if w := s.WindowTokens("m", "s", 24*hourSec); w != 372 { t.Fatalf("24h window want 372, got %d", w) } if w := s.WindowTokens("m", "s", hourSec); w != 2 { t.Fatalf("1h window want 2 (current-bucket row only), got %d", w) } if w := s.WindowTokens("m2", "s", 24*hourSec); w != 10 { t.Fatalf("m2 24h window want 10, got %d", w) } // by_status only from requests (200 x3, 503 x1) — access line must not count if st := s.byStatus[200]; st == nil || st.Reqs != 3 { t.Fatalf("by_status 200 want 3 reqs, got %#v", st) } if s.byStatus[200].Err != 0 || s.byStatus[503] == nil || s.byStatus[503].Reqs != 1 { t.Fatalf("by_status wrong: %#v", s.byStatus) } } // ---- on-demand log loading (plan 阶段 3) ---- // writeAuditFile writes n synthetic request rows (plus some non-request event // rows, which must be skipped) and returns the path. func writeAuditFile(t *testing.T, path string, n int, key string, startMs int64) { t.Helper() f, err := os.Create(path) if err != nil { t.Fatalf("create audit: %v", err) } defer f.Close() for i := 0; i < n; i++ { fmt.Fprintf(f, `{"time":%d,"key":%q,"type":"chat","model":"m","source":"s","prompt_tokens":1,"completion_tokens":1,"latency_ms":5,"ok":true,"status":200}`+"\n", startMs+int64(i)*1000, key) if i%10 == 0 { // an access-log/event row: must never appear as a record fmt.Fprintf(f, `{"obj":"access","time":%d,"path":"/v1/models"}`+"\n", startMs+int64(i)*1000) } } } // TestLoadAuditAggregatesFullHistory is the correctness half of the log work: // the dashboard's totals must cover EVERY audit record, however large the file, // while the in-memory record ring stays bounded. func TestLoadAuditAggregatesFullHistory(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") const n = 20000 writeAuditFile(t, path, n, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) if s.ReplayPartial() { t.Fatal("a readable history must not be reported as partial") } snap := s.Snapshot(0, "") total, _ := snap["total"].(Stat) if total.Reqs != n { t.Fatalf("total reqs = %d, want %d (aggregates must cover the whole file)", total.Reqs, n) } if total.Tokens != int64(n)*2 { // 1 prompt + 1 completion per row t.Fatalf("total tokens = %d, want %d", total.Tokens, n*2) } // ...while the resident ring is capped and the snapshot ships one screen recs, _ := snap["records"].([]Req) if len(recs) == 0 { t.Fatal("the newest records must stay resident") } if len(recs) > firstScreenRecords { t.Fatalf("snapshot shipped %d records, must be capped at the first screen (%d)", len(recs), firstScreenRecords) } s.mu.Lock() ring := len(s.recs) s.mu.Unlock() if ring > defaultRingSize { t.Fatalf("resident ring holds %d records, must be capped at %d", ring, defaultRingSize) } // the ring must hold the NEWEST records, not the first ones scanned newest := recs[len(recs)-1] wantNewest := int64(1_700_000_000_000) + int64(n-1)*1000 if newest.Time != wantNewest { t.Fatalf("newest resident record time = %d, want %d", newest.Time, wantNewest) } } // TestLoadAuditAggregatesAcrossRotatedFiles: history split over rotated .old // files must still add up, and must be replayed oldest-first so the hourly // quota buckets keep the intended trailing window. func TestLoadAuditAggregatesAcrossRotatedFiles(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") writeAuditFile(t, path, 40, "k1", 1_700_000_500_000) // newest writeAuditFile(t, path+".100.old", 40, "k1", 1_700_000_300_000) // older writeAuditFile(t, path+".050.old", 40, "k1", 1_700_000_100_000) // oldest s := NewStats(0) s.LoadAudit(path) snap := s.Snapshot(0, "") total, _ := snap["total"].(Stat) if total.Reqs != 120 { t.Fatalf("total reqs = %d, want 120 across three files", total.Reqs) } recs, _ := snap["records"].([]Req) if len(recs) == 0 { t.Fatal("expected resident records") } // oldest-first replay + newest-last ring => the last resident record is the // newest one in the live file if got := recs[len(recs)-1].Time; got != 1_700_000_500_000+39*1000 { t.Fatalf("newest resident record time = %d, want the live file's last row", got) } } // TestReqRingKeepsNewest pins the ring semantics used by the full-history scan: // bounded size, newest retained, oldest-first output. func TestReqRingKeepsNewest(t *testing.T) { r := newReqRing(3) for i := 1; i <= 7; i++ { r.push(Req{Time: int64(i)}) } got := r.slice() if len(got) != 3 { t.Fatalf("ring len = %d, want 3", len(got)) } for i, want := range []int64{5, 6, 7} { if got[i].Time != want { t.Fatalf("ring[%d] = %d, want %d (oldest-first, newest retained)", i, got[i].Time, want) } } // a partially filled ring returns exactly what was pushed r2 := newReqRing(5) r2.push(Req{Time: 1}) r2.push(Req{Time: 2}) if got := r2.slice(); len(got) != 2 || got[0].Time != 1 || got[1].Time != 2 { t.Fatalf("partial ring = %+v", got) } } // TestLoadAuditFullFileWhenSmall: a small file is loaded completely and is not // flagged partial. func TestLoadAuditFullFileWhenSmall(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") writeAuditFile(t, path, 50, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) if s.ReplayPartial() { t.Fatal("a small file must be replayed in full, not flagged partial") } snap := s.Snapshot(0, "") total, _ := snap["total"].(Stat) if total.Reqs != 50 { t.Fatalf("total reqs = %d, want 50", total.Reqs) } } // TestQuotaWindowsSurviveBoundedRing is the regression guard the plan calls out: // capping the resident record ring must not break token-quota accounting, which // reads the hour buckets rather than the ring. func TestQuotaWindowsSurviveBoundedRing(t *testing.T) { s := NewStats(10) // deliberately tiny ring now := time.Now().UnixMilli() for i := 0; i < 100; i++ { s.Record(Req{Time: now, Key: "k", Model: "m", Source: "s", Prompt: 2, Compl: 3, OK: true, Status: 200}) } if got := s.WindowTokens("m", "s", 24*hourSec); got != 500 { t.Fatalf("window tokens = %d, want 500 (quota accounting must not depend on the ring)", got) } if got := s.ModelTokens("k")["m"]; got != 500 { t.Fatalf("model tokens = %d, want 500", got) } if got := s.KeyTokens("k"); got != 500 { t.Fatalf("key tokens = %d, want 500", got) } } // TestAuditPagePagesBackwards walks the whole history one page at a time and // asserts strict newest-first order with no gaps or repeats. func TestAuditPagePagesBackwards(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") const n = 250 writeAuditFile(t, path, n, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) var seen []Req cursor := "" for i := 0; i < 20; i++ { // bounded loop guards against a cursor that never advances page := s.AuditPage(cursor, 40, "") if len(page.Records) == 0 { break } seen = append(seen, page.Records...) if !page.HasMore { break } if page.Next == cursor { t.Fatal("cursor did not advance") } cursor = page.Next } if len(seen) != n { t.Fatalf("paged %d records, want %d", len(seen), n) } for i := 1; i < len(seen); i++ { if seen[i-1].Time < seen[i].Time { t.Fatalf("records not newest-first at %d: %d then %d", i, seen[i-1].Time, seen[i].Time) } } // newest record first, oldest last if seen[0].Time != 1_700_000_000_000+int64(n-1)*1000 { t.Fatalf("first record time = %d, want the newest", seen[0].Time) } if seen[len(seen)-1].Time != 1_700_000_000_000 { t.Fatalf("last record time = %d, want the oldest", seen[len(seen)-1].Time) } } // TestAuditPageFirstScreenIsSmall: the first page must be exactly one screen, // independent of how large the audit file is. func TestAuditPageFirstScreenIsSmall(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") writeAuditFile(t, path, 5000, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) page := s.AuditPage("", 100, "") if len(page.Records) != 100 { t.Fatalf("first page = %d records, want 100", len(page.Records)) } if !page.HasMore || page.Next == "" { t.Fatal("a large file must report more pages plus a cursor") } } // TestAuditPageFiltersByKey: a user key may only page its own records. func TestAuditPageFiltersByKey(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") writeAuditFile(t, path, 30, "mine", 1_700_000_000_000) f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0644) if err != nil { t.Fatal(err) } for i := 0; i < 30; i++ { fmt.Fprintf(f, `{"time":%d,"key":"other","type":"chat","model":"m","source":"s","ok":true,"status":200}`+"\n", 1_700_000_100_000+int64(i)*1000) } f.Close() s := NewStats(0) s.LoadAudit(path) page := s.AuditPage("", 1000, "mine") if len(page.Records) != 30 { t.Fatalf("filtered page = %d records, want 30", len(page.Records)) } for _, r := range page.Records { if r.Key != "mine" { t.Fatalf("key filter leaked record for %q", r.Key) } } } // TestAuditPageSpansRotatedFiles: paging must continue into .old files instead // of stopping at the live file's start. func TestAuditPageSpansRotatedFiles(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "audit.jsonl") writeAuditFile(t, path, 40, "k1", 1_700_000_500_000) // newest writeAuditFile(t, path+".100.old", 40, "k1", 1_700_000_300_000) // older writeAuditFile(t, path+".050.old", 40, "k1", 1_700_000_100_000) // oldest s := NewStats(0) s.LoadAudit(path) var seen []Req cursor := "" for i := 0; i < 30; i++ { page := s.AuditPage(cursor, 25, "") seen = append(seen, page.Records...) if !page.HasMore { break } cursor = page.Next } if len(seen) != 120 { t.Fatalf("paged %d records across rotated files, want 120", len(seen)) } for i := 1; i < len(seen); i++ { if seen[i-1].Time < seen[i].Time { t.Fatalf("rotation boundary broke the ordering at %d", i) } } } // TestAuditPageRotatedCursor: a cursor whose file rotated away must restart from // the newest page and say so, rather than silently returning nothing. func TestAuditPageRotatedCursor(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") writeAuditFile(t, path, 20, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) page := s.AuditPage("/gone/audit.jsonl:12345", 10, "") if !page.Rotated { t.Fatal("a stale cursor must be reported as rotated") } if len(page.Records) == 0 { t.Fatal("a stale cursor must fall back to the newest page") } } // TestAuditPageEmptyAndNoFile cover the degenerate inputs. func TestAuditPageEmptyAndNoFile(t *testing.T) { s := NewStats(0) if page := s.AuditPage("", 10, ""); len(page.Records) != 0 || page.HasMore { t.Fatalf("no audit path must yield an empty page, got %+v", page) } path := filepath.Join(t.TempDir(), "audit.jsonl") if err := os.WriteFile(path, nil, 0644); err != nil { t.Fatal(err) } s2 := NewStats(0) s2.LoadAudit(path) if page := s2.AuditPage("", 10, ""); len(page.Records) != 0 || page.HasMore { t.Fatalf("empty audit file must yield an empty page, got %+v", page) } } // TestAuditPageHandlesLongLines: a record larger than the reverse-read chunk // must still be decoded (the reader carries partial lines across chunks). func TestAuditPageHandlesLongLines(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") f, err := os.Create(path) if err != nil { t.Fatal(err) } big := strings.Repeat("e", revChunk+1024) // one record spanning >1 chunk fmt.Fprintf(f, `{"time":1,"key":"k","type":"chat","model":"m","source":"s","ok":false,"status":500,"error":%q}`+"\n", big) fmt.Fprintf(f, `{"time":2,"key":"k","type":"chat","model":"m","source":"s","ok":true,"status":200}`+"\n") f.Close() s := NewStats(0) s.LoadAudit(path) page := s.AuditPage("", 10, "") if len(page.Records) != 2 { t.Fatalf("got %d records, want 2 (a >chunk record must survive reverse reads)", len(page.Records)) } if page.Records[0].Time != 2 || page.Records[1].Time != 1 { t.Fatalf("ordering broken: %d then %d", page.Records[0].Time, page.Records[1].Time) } if len(page.Records[1].Err) != len(big) { t.Fatalf("oversized field truncated: %d bytes, want %d", len(page.Records[1].Err), len(big)) } } // TestStreamAuditRecordsIsWindowed: export streaming must honour the time window // and the key filter, and must hand records over one at a time. func TestStreamAuditRecordsIsWindowed(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") writeAuditFile(t, path, 100, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) from := int64(1_700_000_010_000) to := int64(1_700_000_019_000) var got []Req if err := s.StreamAuditRecords(from, to, "", func(r Req) error { got = append(got, r) return nil }); err != nil { t.Fatalf("stream: %v", err) } if len(got) != 10 { t.Fatalf("streamed %d records, want 10 in [%d,%d]", len(got), from, to) } for _, r := range got { if r.Time < from || r.Time > to { t.Fatalf("record outside the window: %d", r.Time) } } } // TestStreamAuditRecordsAbortStops: a callback error (client disconnect) must // stop the walk immediately instead of reading the rest of the history. func TestStreamAuditRecordsAbortStops(t *testing.T) { path := filepath.Join(t.TempDir(), "audit.jsonl") writeAuditFile(t, path, 500, "k1", 1_700_000_000_000) s := NewStats(0) s.LoadAudit(path) seen := 0 stop := fmt.Errorf("client gone") err := s.StreamAuditRecords(0, 0, "", func(Req) error { seen++ if seen == 5 { return stop } return nil }) if err != stop { t.Fatalf("abort must surface the callback error, got %v", err) } if seen != 5 { t.Fatalf("walk continued after abort: %d records", seen) } } // TestAuditRotationThresholds pins the smaller-file/more-files trade-off that // makes reverse paging cheap while keeping the same total budget. func TestAuditRotationThresholds(t *testing.T) { if auditRotateBytes != 16<<20 { t.Fatalf("auditRotateBytes = %d, want 16 MiB", auditRotateBytes) } if auditKeepOld != 16 { t.Fatalf("auditKeepOld = %d, want 16", auditKeepOld) } if total := auditRotateBytes * int64(auditKeepOld); total != 256<<20 { t.Fatalf("total audit budget = %d, want 256 MiB", total) } }