package gateway import ( "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" "strings" "testing" "llmsproxy/internal/config" "llmsproxy/internal/core" ) // newFailingServer serves handler and returns its test server. func newFailingServer(h http.Handler) *httptest.Server { return httptest.NewServer(h) } // THE TEST THAT WAS MISSING. // // Every other plugin test calls Plugins.Fire() directly. That proves the Lua // runtime works, but it says nothing about whether the GATEWAY actually fires // the stages — the wiring in handleChat / singleChat / writeRec could be // deleted and the whole suite would stay green while no hook ever ran in // production. // // So these tests drive real HTTP requests through the real handler and assert // on what the plugin observed. // observed is what a spy plugin recorded. It is written from Lua, so the // counters live in plugin state and are read back through the state API. type spyPayload struct { Stage string Type string Model string Source string OK bool Tokens int64 } // spyPlugin returns Lua source for a plugin that appends every stage it sees to // `state.seen`. It is generated per test so each test gets a clean log. func spyPlugin() string { return ` local plugin = { name = "spy", version = "1.0.0" } plugin.state = { seen = {} } plugin.hooks = { request_start = "on_start", routed = "on_routed", request_end = "on_end", } local function rec(p, stage) table.insert(plugin.state.seen, { stage = stage, type = p.type or "", model = p.model or "", source = p.source or "", ok = p.ok and true or false, tokens = p.prompt_tokens or 0, }) end function plugin.on_start(p) rec(p, "request_start") return nil end function plugin.on_routed(p) rec(p, "routed") return nil end function plugin.on_end(p) rec(p, "request_end") return nil end return plugin ` } // gatewayWithSpy boots a gateway with one mock upstream and the spy plugin // loaded, and returns the gateway plus the upstream URL. func gatewayWithSpy(t *testing.T) (*Gateway, string) { t.Helper() up := mockUpstream() t.Cleanup(up.Close) dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") cfgBody := "listen: :0\n" + "adapter_dir: " + filepath.Join(dir, "adapters") + "\n" + "plugin_dir: " + filepath.Join(dir, "plugins") + "\n" + "runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" + "gateway_keys:\n - sk-test\n" if err := os.WriteFile(cfgPath, []byte(cfgBody), 0600); err != nil { t.Fatal(err) } cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{{ Name: "spyup", BaseURL: up.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "spy-model", Kind: "chat"}}, }} if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatalf("core: %v", err) } t.Cleanup(c.Close) if err := c.Plugins().LoadSource("spy", spyPlugin()); err != nil { t.Fatalf("load spy: %v", err) } g, err := New(c) if err != nil { t.Fatalf("gateway: %v", err) } return g, up.URL } // seenStages reads the spy's log out of its published state. func seenStages(t *testing.T, g *Gateway) []map[string]interface{} { t.Helper() raw := g.core.Plugins().State("spy") if raw == nil { return nil } b, err := json.Marshal(raw) if err != nil { t.Fatal(err) } var st struct { Seen []map[string]interface{} `json:"seen"` } if err := json.Unmarshal(b, &st); err != nil { t.Fatal(err) } return st.Seen } func stageNames(seen []map[string]interface{}) []string { out := make([]string, 0, len(seen)) for _, s := range seen { out = append(out, s["stage"].(string)) } return out } // TestHooksFireOnRealDirectChat: a plain non-streaming chat through the real // handler must produce request_start, routed and request_end — in that order, // with the real source and model filled in. func TestHooksFireOnRealDirectChat(t *testing.T) { g, _ := gatewayWithSpy(t) rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"spy-model","messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusOK { t.Fatalf("chat status=%d body=%s", rr.Code, rr.Body.String()) } seen := seenStages(t, g) got := stageNames(seen) want := []string{"request_start", "routed", "request_end"} if len(got) != 3 { t.Fatalf("stages = %v, want %v (a hook that never fires is the failure "+ "this test exists for)", got, want) } for i := range want { if got[i] != want[i] { t.Fatalf("stage order = %v, want %v", got, want) } } // The payload must be real data, not empty scaffolding. end := seen[2] if end["source"] != "spyup" { t.Errorf("request_end source = %v, want spyup", end["source"]) } if end["model"] != "spy-model" { t.Errorf("request_end model = %v, want spy-model", end["model"]) } if end["ok"] != true { t.Errorf("request_end ok = %v, want true", end["ok"]) } if toInt(t, end["tokens"]) <= 0 { t.Errorf("request_end prompt_tokens = %v, want the mock upstream's real usage", end["tokens"]) } } // TestHooksFireOnRealStreamChat: streaming is a DIFFERENT code path (the record // is written from a deferred call after the stream ends), so it needs its own // assertion. A hook that only worked on the non-streaming path would be a real // bug for agent clients, which are the main streaming users. func TestHooksFireOnRealStreamChat(t *testing.T) { g, _ := gatewayWithSpy(t) rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"spy-model","stream":true,"messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusOK { t.Fatalf("stream status=%d body=%s", rr.Code, rr.Body.String()) } got := stageNames(seenStages(t, g)) if len(got) != 3 { t.Fatalf("stream stages = %v, want all three", got) } if got[0] != "request_start" || got[2] != "request_end" { t.Errorf("stream stage order = %v", got) } } // TestHooksFireOnAutoRequest: the AUTO chain path is a third code path. It also // must fire, and routed must report tier -2 (resolved by the chain) rather than // the direct path's -1. func TestHooksFireOnAutoRequest(t *testing.T) { g, _ := gatewayWithSpy(t) // Register the model in the AUTO chain via the admin API. put := doReq(t, g, http.MethodPut, "/api/auto", `{"rules":[{"model":"spy-model"}]}`) if put.Code != http.StatusOK { t.Fatalf("save auto: %d %s", put.Code, put.Body.String()) } rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusOK { t.Fatalf("auto chat status=%d body=%s", rr.Code, rr.Body.String()) } seen := seenStages(t, g) if len(seen) != 3 { t.Fatalf("AUTO stages = %v, want three", stageNames(seen)) } if seen[0]["model"] != "AUTO" { t.Errorf("request_start model = %v, want the requested \"AUTO\"", seen[0]["model"]) } if seen[1]["model"] != "spy-model" { t.Errorf("routed model = %v, want the RESOLVED model", seen[1]["model"]) } } // TestHooksFireOnFailedRequest is the one that matters most for accounting: a // request that 500s upstream must still reach request_end, or a billing plugin // would never see failed traffic. func TestHooksFireOnFailedRequest(t *testing.T) { // Point at an upstream that always fails. bad := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":"boom"}`)) }) up := newFailingServer(bad) defer up.Close() dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") cfgBody := "listen: :0\n" + "adapter_dir: " + filepath.Join(dir, "adapters") + "\n" + "plugin_dir: " + filepath.Join(dir, "plugins") + "\n" + "runtime_file: " + filepath.Join(dir, "runtime.json") + "\n" + "gateway_keys:\n - sk-test\n" os.WriteFile(cfgPath, []byte(cfgBody), 0600) cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{{ Name: "badup", BaseURL: up.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "bad-model", Kind: "chat"}}, }} if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatalf("core: %v", err) } defer c.Close() if err := c.Plugins().LoadSource("spy", spyPlugin()); err != nil { t.Fatal(err) } g, err := New(c) if err != nil { t.Fatal(err) } rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"bad-model","messages":[{"role":"user","content":"hi"}]}`) if rr.Code == http.StatusOK { t.Fatal("expected the upstream failure to surface as a non-200") } seen := seenStages(t, g) // request_start must fire; routed must NOT (no source was ever selected); // request_end MUST (otherwise billing misses failed traffic). got := stageNames(seen) if len(got) == 0 || got[0] != "request_start" { t.Fatalf("stages = %v, want request_start first", got) } hasEnd := false for _, s := range seen { if s["stage"] == "request_end" { hasEnd = true if s["ok"] != false { t.Error("request_end reported ok=true for a failed request") } } } if !hasEnd { t.Error("a FAILED request never reached request_end; a billing plugin " + "would not see it at all") } if contains(got, "routed") { t.Error("routed fired for a request that never selected a source") } } // TestBrokenPluginDoesNotBreakForwarding is the safety property, asserted // end-to-end: a plugin that throws in EVERY stage must not stop a chat from // being served. func TestBrokenPluginDoesNotBreakForwarding(t *testing.T) { up := mockUpstream() defer up.Close() dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") os.WriteFile(cfgPath, []byte("listen: :0\n"+ "adapter_dir: "+filepath.Join(dir, "adapters")+"\n"+ "plugin_dir: "+filepath.Join(dir, "plugins")+"\n"+ "runtime_file: "+filepath.Join(dir, "runtime.json")+"\n"+ "gateway_keys:\n - sk-test\n"), 0600) cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{{ Name: "u", BaseURL: up.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "m", Kind: "chat"}}, }} if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatal(err) } defer c.Close() broken := ` local plugin = { name = "broken", version = "1.0.0" } plugin.hooks = { request_start = "f", routed = "f", request_end = "f" } function plugin.f(p) error("plugin is broken") end return plugin ` if err := c.Plugins().LoadSource("broken", broken); err != nil { t.Fatalf("load broken: %v", err) } g, err := New(c) if err != nil { t.Fatal(err) } rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"m","messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusOK { t.Fatalf("a broken plugin broke forwarding: status=%d body=%s", rr.Code, rr.Body.String()) } if !strings.Contains(rr.Body.String(), "pong") { t.Errorf("the response is not the upstream's: %s", rr.Body.String()) } // And the failure must be visible rather than silent. if len(c.Plugins().HookErrors()) == 0 { t.Error("a plugin that threw in every stage left no error record") } } // TestPluginListIsServedWithStages is the operator-facing check: an admin // debugging a plugin needs to see which stages are live and whether the hook // file is where the kernel thinks it is. func TestPluginListIsServedWithStages(t *testing.T) { g, _ := gatewayWithSpy(t) rr := doReq(t, g, http.MethodGet, "/api/plugins", "") if rr.Code != http.StatusOK { t.Fatalf("status=%d", rr.Code) } for _, want := range []string{"spy", "request_start", "routed", "request_end", "plugin_dir"} { if !strings.Contains(rr.Body.String(), want) { t.Errorf("/api/plugins response lacks %q", want) } } } func toInt(t *testing.T, v interface{}) int64 { t.Helper() f, ok := v.(float64) if !ok { return 0 } return int64(f) } func contains(list []string, s string) bool { for _, x := range list { if x == s { return true } } return false } // imageUpstream serves an OpenAI-shaped image response. func imageUpstream() *httptest.Server { return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"created":1,"data":[{"b64_json":"aGk="}]}`)) })) } // TestHooksFireOnRealImageRequest: the image path is a THIRD entry point. It // has its own handler and its own scheduler call, so "chat works" says nothing // about it — and image traffic is billable, so a cost plugin that never sees // it under-reports. func TestHooksFireOnRealImageRequest(t *testing.T) { up := imageUpstream() defer up.Close() dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") os.WriteFile(cfgPath, []byte("listen: :0\n"+ "adapter_dir: "+filepath.Join(dir, "adapters")+"\n"+ "plugin_dir: "+filepath.Join(dir, "plugins")+"\n"+ "runtime_file: "+filepath.Join(dir, "runtime.json")+"\n"+ "gateway_keys:\n - sk-test\n"), 0600) cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{{ Name: "imgup", BaseURL: up.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "img-model", Kind: "image"}}, }} if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatalf("core: %v", err) } defer c.Close() if err := c.Plugins().LoadSource("spy", spyPlugin()); err != nil { t.Fatal(err) } g, err := New(c) if err != nil { t.Fatal(err) } rr := doReq(t, g, http.MethodPost, "/v1/images/generations", `{"model":"img-model","prompt":"a cat"}`) if rr.Code != http.StatusOK { t.Fatalf("image status=%d body=%s", rr.Code, rr.Body.String()) } seen := seenStages(t, g) got := stageNames(seen) if len(got) != 3 { t.Fatalf("image stages = %v, want all three (image is a separate handler)", got) } if seen[0]["type"] != "image" { t.Errorf("request_start type = %v, want \"image\"", seen[0]["type"]) } if seen[1]["source"] != "imgup" { t.Errorf("routed source = %v, want imgup", seen[1]["source"]) } } // TestRejectedChatStillFiresRequestStart pins the ordering decision: request_start // fires BEFORE the quota/scope gates, so a plugin auditing traffic sees rejected // requests too. A plugin that only ever saw served requests could not compute a // true request rate. func TestRejectedChatStillFiresRequestStart(t *testing.T) { up := mockUpstream() defer up.Close() dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") os.WriteFile(cfgPath, []byte("listen: :0\n"+ "adapter_dir: "+filepath.Join(dir, "adapters")+"\n"+ "plugin_dir: "+filepath.Join(dir, "plugins")+"\n"+ "runtime_file: "+filepath.Join(dir, "runtime.json")+"\n"+ "gateway_keys:\n - sk-test\n"), 0600) cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{{ Name: "u", BaseURL: up.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "m", Kind: "chat"}}, }} if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatal(err) } defer c.Close() if err := c.Plugins().LoadSource("spy", spyPlugin()); err != nil { t.Fatal(err) } g, err := New(c) if err != nil { t.Fatal(err) } // A model the gateway does not know => 404, rejected before any routing. rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"no-such-model","messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusNotFound { t.Fatalf("expected 404 for an unknown model, got %d", rr.Code) } got := stageNames(seenStages(t, g)) if !contains(got, "request_start") { t.Errorf("stages = %v; a REJECTED request must still fire request_start so a "+ "plugin can count real traffic, not just served traffic", got) } } // TestChainStepReachesPluginOnDegradation is the end-to-end proof for the // AUTO trace: a request that had to drop from tier 1 to tier 2 must be visible // to a plugin as a tier_skip followed by a selected, and request_end must carry // tier_served=2. // // Before the trace existed the plugin saw only tier=-2 ("resolved by the // chain") and could not tell a degradation from a clean tier-1 hit — which is // the whole question a priority chain exists to answer. func TestChainStepReachesPluginOnDegradation(t *testing.T) { // tier 1's source always fails, so the walk must drop to tier 2. bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) _, _ = w.Write([]byte(`{"error":"boom"}`)) })) defer bad.Close() good := mockUpstream() defer good.Close() dir := t.TempDir() cfgPath := filepath.Join(dir, "config.yaml") os.WriteFile(cfgPath, []byte("listen: :0\n"+ "adapter_dir: "+filepath.Join(dir, "adapters")+"\n"+ "plugin_dir: "+filepath.Join(dir, "plugins")+"\n"+ "runtime_file: "+filepath.Join(dir, "runtime.json")+"\n"+ "gateway_keys:\n - sk-test\n"), 0600) cfg, err := config.Load(cfgPath) if err != nil { t.Fatal(err) } cfg.Sources = []config.Source{ {Name: "t1", BaseURL: bad.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "hi-tier", Kind: "chat"}}}, {Name: "t2", BaseURL: good.URL, Adapter: "openai", APIKey: "sk-x", Models: []config.Model{{ID: "lo-tier", Kind: "chat"}}}, } if err := cfg.ApplyDefaults(); err != nil { t.Fatal(err) } c, err := core.NewFromConfig(cfg) if err != nil { t.Fatal(err) } defer c.Close() // A spy that records chain_step events too. sp := ` local plugin = { name = "walker", version = "1.0.0" } plugin.state = { steps = {}, ends = {} } plugin.hooks = { chain_step = "step", request_end = "fin" } function plugin.step(p) table.insert(plugin.state.steps, { kind = p.kind, tier = p.tier, source = p.source, model = p.model, reason = p.reason }) return nil end function plugin.fin(p) plugin.state.ends[#plugin.state.ends + 1] = { tier_served = p.tier_served, degraded = p.degraded, walk = p.chain_walk, source = p.source, model = p.model, } return nil end return plugin ` if err := c.Plugins().LoadSource("walker", sp); err != nil { t.Fatal(err) } g, err := New(c) if err != nil { t.Fatal(err) } // Two tiers, both in the chain. if rr := doReq(t, g, http.MethodPut, "/api/auto", `{"rules":[{"model":"hi-tier","source":"t1","tier":1},{"model":"lo-tier","source":"t2","tier":2}]}`); rr.Code != 200 { t.Fatalf("save auto: %d %s", rr.Code, rr.Body.String()) } rr := doReq(t, g, http.MethodPost, "/v1/chat/completions", `{"model":"AUTO","messages":[{"role":"user","content":"hi"}]}`) if rr.Code != http.StatusOK { t.Fatalf("chat = %d %s", rr.Code, rr.Body.String()) } raw := c.Plugins().State("walker") b, _ := json.Marshal(raw) var st struct { Steps []struct { Kind string `json:"kind"` Tier int `json:"tier"` Source string `json:"source"` Model string `json:"model"` } `json:"steps"` Ends []struct { TierServed int `json:"tier_served"` Degraded bool `json:"degraded"` Source string `json:"source"` Model string `json:"model"` } `json:"ends"` } if err := json.Unmarshal(b, &st); err != nil { t.Fatalf("decode: %v (%s)", err, string(b)) } if len(st.Steps) < 2 { t.Fatalf("chain_step events = %+v, want at least a slot_fail and a selected", st.Steps) } if st.Steps[0].Kind != "slot_fail" || st.Steps[0].Tier != 1 { t.Errorf("first step = %+v, want slot_fail on tier 1", st.Steps[0]) } last := st.Steps[len(st.Steps)-1] if last.Kind != "selected" || last.Tier != 2 { t.Errorf("last step = %+v, want selected on tier 2", last) } if len(st.Ends) != 1 { t.Fatalf("request_end count = %d, want 1", len(st.Ends)) } if st.Ends[0].TierServed != 2 { t.Errorf("tier_served = %d, want 2", st.Ends[0].TierServed) } if !st.Ends[0].Degraded { t.Error("degraded = false, but the request dropped a tier") } if st.Ends[0].Model != "lo-tier" { t.Errorf("served model = %q, want lo-tier", st.Ends[0].Model) } }