package gateway // Mutation harness for the discovery code. Kept in the tree (not in a shell // script) so the -run filter is a Go string literal rather than something the // shell can mangle into "[no tests to run]" — which is exactly how an earlier // version of this harness reported four surviving mutants that were simply // never executed. // // Each mutant flips one behaviour; TestMutationHarness_Fails proves the // harness itself is armed. import ( "encoding/json" "strings" "testing" ) // discoveryTestFilter must cover every assertion that guards the behaviour // below. Narrowing it is how mutants get to "survive" for free. const discoveryTestFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates)" func TestDiscoveryFilterRunsRealTests(t *testing.T) { if discoveryTestFilter == "" { t.Fatal("filter is empty; every mutant would survive trivially") } // The whole point: the catalog parser test must be inside the filter, or a // mutant that stops reading nested metadata survives with a green run. for _, group := range []string{ "ParseModelCatalog", // nested metadata + non-catalog rejection "DiscoveredModelKind", // kind must follow upstream metadata "Discover", // URL fallback + endpoint composition "ComposeEndpoint", // version segment exactly once "IsVersionSegment", // "v1beta" vs "vision" } { if !strings.Contains(discoveryTestFilter, group) { t.Errorf("filter %q skips %s tests; mutants in that area survive", discoveryTestFilter, group) } } } // TestMutantsAreDetectable is the harness's own regression guard: it runs the // filter's worth of assertions against deliberately broken inputs and requires // each to be rejected. If a future edit makes one of these pass, the // corresponding protection is gone. func TestMutantsAreDetectable(t *testing.T) { t.Run("image classification follows upstream metadata, not the model name", func(t *testing.T) { // The mutant: drop the output-modality check, so every model is chat. m := upstreamModel{ID: "gpt-image-2"} m.Architecture.OutputModalities = []string{"image"} if got := discoveredModelKind(m); got != "image" { t.Errorf("mutant survived: got %q, want \"image\"", got) } }) t.Run("nested metadata is read", func(t *testing.T) { var ms []upstreamModel if err := json.Unmarshal([]byte(realCatalogBody), &ms); err == nil { t.Fatal("fixture must be a wrapper object, not a bare array") } var wrapper struct { Data []upstreamModel `json:"data"` } if err := json.Unmarshal([]byte(realCatalogBody), &wrapper); err != nil { t.Fatal(err) } if len(wrapper.Data) == 0 { t.Fatal("fixture parsed to zero models") } // The mutant: skip copyFields(m.Metadata) entirely. meta := modelMeta(wrapper.Data[0]) if meta == nil { t.Fatal("mutant survived: metadata is nil, so cost/context_limit are lost") } for _, k := range []string{"context_limit", "cost", "context_budget"} { if _, ok := meta[k]; !ok { t.Errorf("mutant survived: %s dropped from nested metadata", k) } } }) t.Run("non-catalog responses are rejected", func(t *testing.T) { for _, body := range []string{`login`, `null`, `{"error":{"message":"x"}}`, `{}`} { if _, err := parseModelCatalog([]byte(body)); err == nil { t.Errorf("mutant survived: %q accepted as a catalog", body) } } }) t.Run("version segments are distinguished from real path segments", func(t *testing.T) { if !isVersionSegment("v1beta") { t.Error("mutant survived: v1beta not treated as a version") } for _, seg := range []string{"vision", "vertex", "v1x"} { if isVersionSegment(seg) { t.Errorf("mutant survived: %q wrongly treated as a version", seg) } } }) t.Run("endpoint composition never duplicates the version segment", func(t *testing.T) { // The mutant: never move the segment onto the endpoint. if got := composeEndpoint("/v1/models", "https://h", "https://h/v1"); got != "/v1/chat/completions" { t.Errorf("mutant survived: endpoint = %q", got) } // The other mutant: always move it, even when the base keeps it. if got := composeEndpoint("/models", "https://h/v1", "https://h/v1"); got != "" { t.Errorf("mutant survived: endpoint = %q would double /v1", got) } }) t.Run("image_models is never nil", func(t *testing.T) { res := &discoveryResult{Models: []string{"a"}, ImageModels: nil} b, _ := json.Marshal(res) if strings.Contains(string(b), `"image_models":null`) { t.Errorf("mutant survived: %s", b) } }) t.Run("discovery falls back across URL forms", func(t *testing.T) { cands := discoveryCandidates("https://h/v1") if len(cands) < 3 { t.Errorf("mutant survived: only %d candidates, fallback search removed", len(cands)) } }) t.Run("only the catalog is fetched, never each model", func(t *testing.T) { // Guarding a NON-mutation: the design decision that discovery must not // spend money. If a future change adds per-model probes, the counting // test below catches it; this assertion documents the intent. if discoveryModelsPath != "/v1/models" { t.Errorf("catalog path changed to %q", discoveryModelsPath) } }) }