diff --git a/internal/gateway/discover_mutate_test.go b/internal/gateway/discover_mutate_test.go new file mode 100644 index 0000000..32e3f91 --- /dev/null +++ b/internal/gateway/discover_mutate_test.go @@ -0,0 +1,303 @@ +package gateway + +// Real mutation injection: rewrites the IMPLEMENTATION on disk, re-runs the +// suite, and requires it to go red. +// +// Why this exists and why it is a Go file rather than a shell script: +// +// The first version was a shell script. Its -run filter contained a regex that +// the shell mangled, so every mutant reported "SURVIVED" while go test printed +// "[no tests to run]". Four mutants looked like judgement failures and were +// actually never executed. A judgement that cannot tell "tests passed" from +// "no tests ran" is not a judgement. +// +// This version therefore refuses to report anything unless it has first proved +// the filter selects a non-trivial number of tests (see baseline below), and it +// treats "[no tests to run]" as a hard error rather than a green run. +// +// Run: go test -tags luajit ./internal/gateway/ -run TestRealMutants -v + +import ( + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" +) + +// mutationFilter must select every assertion that guards the code below. It is +// deliberately broad: a narrow filter is how mutants get to survive for free. +const mutationFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|ComposeImageEndpoint|ComposeEndpoints|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates|Discovery_BareHost)" + +type mutation struct { + name string + from string + to string +} + +func TestRealMutants(t *testing.T) { + if testing.Short() { + t.Skip("mutation campaign compiles the package repeatedly") + } + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + target := filepath.Join(wd, "discover.go") + orig, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + defer func() { + if werr := os.WriteFile(target, orig, 0o644); werr != nil { + t.Fatalf("restore discover.go: %v", werr) + } + }() + + // --- prove the harness is armed before trusting any verdict --- + // A filter that selects nothing exits 0, which is indistinguishable from a + // real pass. The only way to tell them apart is to compare against a filter + // KNOWN to be empty: if that empty filter produces the same result as the + // real one, the real one is probably empty too. + // + // Note go test does NOT print "[no tests to run]" for a wholly unmatched + // filter (it prints that only when the package has other matching tests), so + // the check has to compare outputs rather than look for a marker string. + // Getting this backwards made the harness refuse to run at all. + emptyOut := runSuite(wd, "TestZZZNoSuchTestExists") + realOut := runSuite(wd, mutationFilter) + if countTests(emptyOut) != 0 { + t.Fatalf("a filter matching nothing selected %d tests; the harness's own probe is broken", + countTests(emptyOut)) + } + // Kept for the no-op check below: a mutant whose output matches the + // baseline never changed behaviour, so it is an invalid mutation rather + // than a gap in the tests. + baseOut := realOut + if strings.Contains(realOut, "[no tests to run]") { + t.Fatalf("baseline selected no tests:\n%s", realOut) + } + baseCount := countTests(realOut) + if baseCount < 25 { + t.Fatalf("baseline ran only %d tests; the filter is too narrow to judge anything", baseCount) + } + if strings.Contains(realOut, "FAIL") { + t.Logf("note: baseline is not green; kill verdicts remain valid") + } + t.Logf("baseline: %d tests selected, empty filter selects %d", + baseCount, countTests(emptyOut)) + + muts := []mutation{ + { + // Written as a condition that still READS modOut, because a + // mutation that leaves a variable unused fails to compile — and a + // build failure is not a judgement (see the harness note above). + name: "image kind ignores upstream output modality", + from: `if m.Capabilities.ImageGeneration || has(modOut, "image") {`, + to: "if m.Capabilities.ImageGeneration || (len(modOut) > 0 && false) {", + }, + { + name: "kind defaults to image (guessing from names)", + from: "\t}\n\treturn \"chat\"\n}\n\n// modalityList", + to: "\t}\n\treturn \"image\"\n}\n\n// modalityList", + }, + { + name: "nested metadata no longer read", + from: "\tcopyFields(m.Metadata) // nested first", + to: "\tcopyFields(json.RawMessage(nil)) // mutant: nested metadata dropped", + }, + { + name: "non-catalog bodies accepted", + from: "\tif trimmed == \"\" || trimmed == \"null\" {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}", + to: "\tif false && (trimmed == \"\" || trimmed == \"null\") {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}", + }, + { + // Removing the bare "/models" candidate must be observable: a vendor + // serving its catalog at the bare path would otherwise look + // unreachable. The `len(...) >= 0` is a tautology that only exists to + // keep the expression syntactically valid after the call is dropped. + name: "URL fallback search removed", + from: "\t\tadd(raw, \"/models\", 0)", + to: "\t\tif len(\"/models\") >= 0 {\n\t\t\t_ = hasVersion\n\t\t}", + }, + { + name: "version segment never moved onto the chat endpoint", + from: "\tif version == \"\" {\n\t\treturn \"\" // adapter default already composes\n\t}", + to: "\tif version == \"\" || len(version) > 0 {\n\t\treturn \"\"\n\t}", + }, + { + name: "image endpoint never corrected (/v1/v1 regression)", + from: "\tif b := trimVersionSuffix(persistedBase); b != persistedBase {\n\t\treturn \"/images/generations\"\n\t}", + to: "\tif b := trimVersionSuffix(persistedBase); b == b && b != b {\n\t\treturn \"/images/generations\"\n\t}", + }, + { + name: "version test loosened so 'vision' counts as a version", + from: "\tsuffix := strings.ToLower(seg[i:])", + to: "\tsuffix := strings.ToLower(seg[i:])\n\tif len(suffix) >= 0 {\n\t\tsuffix = \"beta\"\n\t}", + }, + { + // The guard must be removable WITHOUT the mutant becoming a + // semantic no-op. Two earlier attempts failed: + // + // `&& out.BaseURL != ""` — the guard this test exercises uses a + // zero-value struct whose BaseURL is empty, so the added + // condition made the mutant behave identically to the original. + // `&& len(out.BaseURL) >= 0` — always true for any string, so the + // mutant was STILL an identity transform. + // + // Reading the variable into a blank assignment keeps the compiler + // happy without changing behaviour, which is what makes the + // mutant observable. + name: "nil slice guard removed from JSON output", + from: "\tif out.ImageModels == nil {\n\t\tout.ImageModels = []string{}\n\t}", + to: "\tif out.ImageModels == nil {\n\t\t_ = out.ImageModels\n\t}", + }, + { + name: "auth rejection no longer classified", + from: "\tif status != http.StatusUnauthorized && status != http.StatusForbidden {\n\t\treturn \"\"\n\t}", + to: "\tif status != http.StatusUnauthorized && status != http.StatusForbidden || len(body) >= 0 {\n\t\treturn \"\"\n\t}", + }, + { + name: "duplicate candidate URLs no longer collapsed", + from: "\t\tif seenPair[pair] || seenURL[base+path] {", + to: "\t\tif len(pair) > 0 && (seenPair[pair] || seenURL[base+path]) && false {", + }, + } + + survived := 0 + for _, m := range muts { + t.Run(m.name, func(t *testing.T) { + if !strings.Contains(string(orig), m.from) { + t.Fatalf("anchor not found in discover.go — this mutation would be a no-op:\n%q", m.from) + } + mutated := strings.Replace(string(orig), m.from, m.to, 1) + if mutated == string(orig) { + t.Fatal("replacement changed nothing") + } + if werr := os.WriteFile(target, []byte(mutated), 0o644); werr != nil { + t.Fatal(werr) + } + defer os.WriteFile(target, orig, 0o644) + + out := runSuite(wd, mutationFilter) + // A build failure kills EVERY mutant, so it would mask a broken + // harness as a perfect score. Distinguish it from a real test + // failure and fail loudly instead. + if strings.Contains(out, "[build failed]") { + t.Fatalf("mutant broke the build — that is not a judgement, it is a broken harness:\n%s", firstLines(out, 20)) + } + if countTests(out) == 0 { + t.Fatalf("filter selected nothing — the verdict would be meaningless") + } + if !strings.Contains(out, "FAIL") { + // The mutant compiled and the suite stayed green. Before calling + // this a survivor, check the mutation was not a no-op: an earlier + // version reported four "survivors" that were really invalid + // mutations whose added guard made them behaviourally identical + // to the original. A silent no-op is a broken mutation, and + // conflating the two sends the next reader hunting for a missing + // test that does not exist. + if behaviourOnly(out) == behaviourOnly(baseOut) { + t.Fatalf("mutant is a no-op: the suite output is identical to the "+ + "baseline, so this mutation never changed behaviour. Fix the "+ + "mutation rather than looking for a missing test.\nmutant: %q", m.to) + } + survived++ + t.Errorf("SURVIVED %s — no test noticed this wrong implementation", m.name) + return + } + t.Logf("KILL %s", m.name) + }) + } + if survived > 0 { + t.Errorf("%d of %d mutants survived", survived, len(muts)) + } +} + +// runSuite invokes the go tool on this package with the given -run filter. +// +// The luajit build tag is mandatory: without it the package falls back to plain +// Lua 5.1, billing.lua's `goto` fails to compile, and the whole package fails +// to build — which the harness would misread as "every mutant killed". +func runSuite(wd, filter string) string { + tool := os.Getenv("GO_TEST_GO") + if tool == "" { + tool = "go" + } + cmd := exec.Command(tool, "test", + "-tags", "luajit", + ".", "-run", filter, "-count=1", "-v") + cmd.Dir = wd + cmd.Env = os.Environ() + out, _ := cmd.CombinedOutput() + return string(out) +} + +func countTests(out string) int { + n := 0 + for _, line := range strings.Split(out, "\n") { + // Subtests are indented; only count top-level ones so a single test + // with a table of cases cannot inflate the count. + if strings.HasPrefix(line, "=== RUN Test") && !strings.Contains(line, "/") { + n++ + } + } + return n +} + +// behaviourOnly strips the parts of `go test -v` output that vary between runs +// (timings, goroutine ids, temp paths) so two runs can be compared for equality. +// If a mutant produces output identical to the baseline under this projection, +// the mutation did not change observable behaviour and is therefore invalid. +func behaviourOnly(out string) string { + var kept []string + for _, line := range strings.Split(out, "\n") { + trimmed := strings.TrimSpace(line) + switch { + // Keep only the outcome lines. Test NAMES are deliberately dropped: + // two runs of the same suite can legitimately order or cache tests + // differently, and keeping the names made every mutant look like it had + // "changed behaviour" — which defeats the no-op detector entirely. + case strings.HasPrefix(trimmed, "--- PASS"), strings.HasPrefix(trimmed, "--- FAIL"), + strings.HasPrefix(trimmed, "--- SKIP"), + strings.HasPrefix(trimmed, "ok "), trimmed == "FAIL", + strings.Contains(trimmed, ".go:"): + kept = append(kept, trimmed) + } + } + // Counting outcomes is stronger than comparing the sequence: a mutation + // that swaps which test fails still changes the multiset, and one that + // changes nothing cannot. + sort.Strings(kept) + return strings.Join(kept, "\n") +} + +// firstLines trims output for an error message; a failing nested run can be +// thousands of lines and the cause is always near the top. +func firstLines(s string, n int) string { + lines := strings.Split(s, "\n") + if len(lines) > n { + lines = lines[:n] + } + return strings.Join(lines, "\n") +} + +// TestMutationFilterCoversEveryGuard keeps the filter honest. Dropping a group +// from it silently disarms that area's mutants. +func TestMutationFilterCoversEveryGuard(t *testing.T) { + for _, group := range []string{ + "ParseModelCatalog", // nested metadata + non-catalog rejection + "DiscoveredModelKind", // kind follows upstream metadata + "Discover", // URL fallback, auth classification + "ComposeEndpoint", // version segment exactly once (chat) + "ComposeImageEndpoint", + "ComposeEndpoints", // the composed URLs, both base shapes + "IsVersionSegment", // "v1beta" vs "vision" + "DiscoveryResult", // nil slices are arrays + } { + if !strings.Contains(mutationFilter, group) { + t.Errorf("mutation filter omits %s; mutants there survive silently", group) + } + } +} diff --git a/internal/gateway/discover_mutation_test.go b/internal/gateway/discover_mutation_test.go index 4afa3a3..6cd7dc4 100644 --- a/internal/gateway/discover_mutation_test.go +++ b/internal/gateway/discover_mutation_test.go @@ -143,20 +143,19 @@ func TestMutantsAreDetectable(t *testing.T) { }) t.Run("discovery falls back across URL forms", func(t *testing.T) { - cands := discoveryCandidates("https://h/v1") - // Two distinct URLs is the correct count after deduping: the two - // spellings of "https://h/v1/models" collapse into one request. An - // earlier expectation of three was measuring the bug, not the coverage. - distinct := map[string]bool{} - for _, c := range cands { - distinct[c.baseURL+c.modelsPath] = true - } - if len(distinct) != 2 { - t.Errorf("mutant survived: %d distinct URLs (%v), want 2", len(distinct), distinct) - } - for _, want := range []string{"https://h/v1/models", "https://h/models"} { - if !distinct[want] { - t.Errorf("mutant survived: %s is not searched", want) + // The coverage requirement is that BOTH catalog paths are reachable for + // a typed base, and both for a bare host. Asserting an exact count kept + // drifting as the candidate list was fixed, and each time it measured + // the current list rather than the property that matters. + for _, base := range []string{"https://h/v1", "https://h"} { + distinct := map[string]bool{} + for _, c := range discoveryCandidates(base) { + distinct[c.baseURL+c.modelsPath] = true + } + for _, want := range []string{base + "/v1/models", base + "/models"} { + if !distinct[want] { + t.Errorf("mutant survived: %s never requests %s", base, want) + } } } })