ModelRouter/internal/gateway/discover_mutate_test.go

package gateway

// Real mutation injection: rewrites the IMPLEMENTATION on disk, re-runs the
// suite, and requires it to go red.
//
// Why this exists and why it is a Go file rather than a shell script:
//
// The first version was a shell script. Its -run filter contained a regex that
// the shell mangled, so every mutant reported "SURVIVED" while go test printed
// "[no tests to run]". Four mutants looked like judgement failures and were
// actually never executed. A judgement that cannot tell "tests passed" from
// "no tests ran" is not a judgement.
//
// This version therefore refuses to report anything unless it has first proved
// the filter selects a non-trivial number of tests (see baseline below), and it
// treats "[no tests to run]" as a hard error rather than a green run.
//
// It also must not be run against a DIRTY tree: the mutant is written over
// discover.go, and a leftover mutant from a killed run would make the baseline
// green against broken code while every verdict came from the wrong source.
// See the guard at the top of TestRealMutants.
//
// Run: go test -tags luajit ./internal/gateway/ -run TestRealMutants -v

import (
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"
)

// mutationFilter must select every assertion that guards the code below. It is
// deliberately broad: a narrow filter is how mutants get to survive for free.
const mutationFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|ComposeImageEndpoint|ComposeEndpoints|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates|Discovery_BareHost)"

type mutation struct {
	name string
	from string
	to   string
}

func TestRealMutants(t *testing.T) {
	if testing.Short() {
		t.Skip("mutation campaign compiles the package repeatedly")
	}
	wd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	target := filepath.Join(wd, "discover.go")
	orig, err := os.ReadFile(target)
	if err != nil {
		t.Fatal(err)
	}

	// Refuse to run against a dirty tree.
	//
	// The mutant is written OVER discover.go, so on a dirty file the "original"
	// captured above is not the committed source — the baseline then runs against
	// whatever was there, and a killed run (a background-task timeout, a Ctrl-C)
	// leaves a mutant behind that the NEXT run treats as the baseline. Both make
	// every verdict meaningless while the harness still reports a clean score.
	//
	// Refusing rather than restoring: this cannot tell "leftover mutant" apart from
	// "the developer is mid-edit", and silently reverting real work is worse than
	// stopping.
	if dirty, derr := mutationTargetIsDirty(wd, "discover.go"); derr == nil && dirty {
		t.Fatalf("%s has uncommitted changes — commit or stash them first. A mutant "+
			"left behind by a killed run would otherwise become the baseline, and "+
			"every verdict below would be about code nobody wrote.", target)
	}
	defer func() {
		if werr := os.WriteFile(target, orig, 0o644); werr != nil {
			t.Fatalf("restore discover.go: %v", werr)
		}
	}()

	// --- prove the harness is armed before trusting any verdict ---
	// A filter that selects nothing exits 0, which is indistinguishable from a
	// real pass. The only way to tell them apart is to compare against a filter
	// KNOWN to be empty: if that empty filter produces the same result as the
	// real one, the real one is probably empty too.
	//
	// Note go test does NOT print "[no tests to run]" for a wholly unmatched
	// filter (it prints that only when the package has other matching tests), so
	// the check has to compare outputs rather than look for a marker string.
	// Getting this backwards made the harness refuse to run at all.
	emptyOut := runSuite(wd, "TestZZZNoSuchTestExists")
	realOut := runSuite(wd, mutationFilter)
	if countTests(emptyOut) != 0 {
		t.Fatalf("a filter matching nothing selected %d tests; the harness's own probe is broken",
			countTests(emptyOut))
	}
	// Kept for the no-op check below: a mutant whose output matches the
	// baseline never changed behaviour, so it is an invalid mutation rather
	// than a gap in the tests.
	baseOut := realOut
	if strings.Contains(realOut, "[no tests to run]") {
		t.Fatalf("baseline selected no tests:\n%s", realOut)
	}
	baseCount := countTests(realOut)
	if baseCount < 25 {
		t.Fatalf("baseline ran only %d tests; the filter is too narrow to judge anything", baseCount)
	}
	if strings.Contains(realOut, "FAIL") {
		t.Logf("note: baseline is not green; kill verdicts remain valid")
	}
	t.Logf("baseline: %d tests selected, empty filter selects %d",
		baseCount, countTests(emptyOut))

	muts := []mutation{
		{
			// Written as a condition that still READS modOut, because a
			// mutation that leaves a variable unused fails to compile — and a
			// build failure is not a judgement (see the harness note above).
			name: "image kind ignores upstream output modality",
			from: `if m.Capabilities.ImageGeneration || has(modOut, "image") {`,
			to:   "if m.Capabilities.ImageGeneration || (len(modOut) > 0 && false) {",
		},
		{
			name: "kind defaults to image (guessing from names)",
			from: "\t}\n\treturn \"chat\"\n}\n\n// modalityList",
			to:   "\t}\n\treturn \"image\"\n}\n\n// modalityList",
		},
		{
			name: "nested metadata no longer read",
			from: "\tcopyFields(m.Metadata)      // nested first",
			to:   "\tcopyFields(json.RawMessage(nil)) // mutant: nested metadata dropped",
		},
		{
			name: "non-catalog bodies accepted",
			from: "\tif trimmed == \"\" || trimmed == \"null\" {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}",
			to:   "\tif false && (trimmed == \"\" || trimmed == \"null\") {\n\t\treturn nil, fmt.Errorf(\"empty response\")\n\t}",
		},
		{
			// Removing the bare "/models" candidate must be observable: a vendor
			// serving its catalog at the bare path would otherwise look
			// unreachable. The `len(...) >= 0` is a tautology that only exists to
			// keep the expression syntactically valid after the call is dropped.
			name: "URL fallback search removed",
			from: "\t\tadd(raw, \"/models\", 0)",
			to:   "\t\tif len(\"/models\") >= 0 {\n\t\t\t_ = hasVersion\n\t\t}",
		},
		{
			name: "version segment never moved onto the chat endpoint",
			from: "\tif version == \"\" {\n\t\treturn \"\" // adapter default already composes\n\t}",
			to:   "\tif version == \"\" || len(version) > 0 {\n\t\treturn \"\"\n\t}",
		},
		{
			name: "image endpoint never corrected (/v1/v1 regression)",
			from: "\tif b := trimVersionSuffix(persistedBase); b != persistedBase {\n\t\treturn \"/images/generations\"\n\t}",
			to:   "\tif b := trimVersionSuffix(persistedBase); b == b && b != b {\n\t\treturn \"/images/generations\"\n\t}",
		},
		{
			name: "version test loosened so 'vision' counts as a version",
			from: "\tsuffix := strings.ToLower(seg[i:])",
			to:   "\tsuffix := strings.ToLower(seg[i:])\n\tif len(suffix) >= 0 {\n\t\tsuffix = \"beta\"\n\t}",
		},
		{
			// The guard must be removable WITHOUT the mutant becoming a
			// semantic no-op. Two earlier attempts failed:
			//
			//   `&& out.BaseURL != ""`  — the guard this test exercises uses a
			//     zero-value struct whose BaseURL is empty, so the added
			//     condition made the mutant behave identically to the original.
			//   `&& len(out.BaseURL) >= 0` — always true for any string, so the
			//     mutant was STILL an identity transform.
			//
			// Reading the variable into a blank assignment keeps the compiler
			// happy without changing behaviour, which is what makes the
			// mutant observable.
			name: "nil slice guard removed from JSON output",
			from: "\tif out.ImageModels == nil {\n\t\tout.ImageModels = []string{}\n\t}",
			to:   "\tif out.ImageModels == nil {\n\t\t_ = out.ImageModels\n\t}",
		},
		{
			name: "auth rejection no longer classified",
			from: "\tif status != http.StatusUnauthorized && status != http.StatusForbidden {\n\t\treturn \"\"\n\t}",
			to:   "\tif status != http.StatusUnauthorized && status != http.StatusForbidden || len(body) >= 0 {\n\t\treturn \"\"\n\t}",
		},
		{
			name: "duplicate candidate URLs no longer collapsed",
			from: "\t\tif seenPair[pair] || seenURL[base+path] {",
			to:   "\t\tif len(pair) > 0 && (seenPair[pair] || seenURL[base+path]) && false {",
		},
	}

	survived := 0
	for _, m := range muts {
		t.Run(m.name, func(t *testing.T) {
			if !strings.Contains(string(orig), m.from) {
				t.Fatalf("anchor not found in discover.go — this mutation would be a no-op:\n%q", m.from)
			}
			mutated := strings.Replace(string(orig), m.from, m.to, 1)
			if mutated == string(orig) {
				t.Fatal("replacement changed nothing")
			}
			if werr := os.WriteFile(target, []byte(mutated), 0o644); werr != nil {
				t.Fatal(werr)
			}
			defer os.WriteFile(target, orig, 0o644)

			out := runSuite(wd, mutationFilter)
			// A build failure kills EVERY mutant, so it would mask a broken
			// harness as a perfect score. Distinguish it from a real test
			// failure and fail loudly instead.
			if strings.Contains(out, "[build failed]") {
				t.Fatalf("mutant broke the build — that is not a judgement, it is a broken harness:\n%s", firstLines(out, 20))
			}
			if countTests(out) == 0 {
				t.Fatalf("filter selected nothing — the verdict would be meaningless")
			}
			if !strings.Contains(out, "FAIL") {
				// The mutant compiled and the suite stayed green. Before calling
				// this a survivor, check the mutation was not a no-op: an earlier
				// version reported four "survivors" that were really invalid
				// mutations whose added guard made them behaviourally identical
				// to the original. A silent no-op is a broken mutation, and
				// conflating the two sends the next reader hunting for a missing
				// test that does not exist.
				if behaviourOnly(out) == behaviourOnly(baseOut) {
					t.Fatalf("mutant is a no-op: the suite output is identical to the "+
						"baseline, so this mutation never changed behaviour. Fix the "+
						"mutation rather than looking for a missing test.\nmutant: %q", m.to)
				}
				survived++
				t.Errorf("SURVIVED %s — no test noticed this wrong implementation", m.name)
				return
			}
			t.Logf("KILL  %s", m.name)
		})
	}
	if survived > 0 {
		t.Errorf("%d of %d mutants survived", survived, len(muts))
	}
}

// runSuite invokes the go tool on this package with the given -run filter.
//
// The luajit build tag is mandatory: without it the package falls back to plain
// Lua 5.1, billing.lua's `goto` fails to compile, and the whole package fails
// to build — which the harness would misread as "every mutant killed".
func runSuite(wd, filter string) string {
	tool := os.Getenv("GO_TEST_GO")
	if tool == "" {
		tool = "go"
	}
	cmd := exec.Command(tool, "test",
		"-tags", "luajit",
		".", "-run", filter, "-count=1", "-v")
	cmd.Dir = wd
	cmd.Env = os.Environ()
	out, _ := cmd.CombinedOutput()
	return string(out)
}

// mutationTargetIsDirty reports whether a file under internal/gateway has
// uncommitted changes, so the harness can refuse to run instead of treating a
// leftover mutant as the baseline.
//
// git runs from the package directory with a PATH RELATIVE to it. An earlier
// version of the sibling script passed a repo-relative pathspec while running in
// the package directory; git matched nothing and the guard never fired.
func mutationTargetIsDirty(wd, name string) (bool, error) {
	cmd := exec.Command("git", "diff", "--name-only", "--", "./"+name)
	cmd.Dir = wd
	out, err := cmd.Output()
	if err != nil {
		return false, err
	}
	return strings.TrimSpace(string(out)) != "", nil
}

func countTests(out string) int {
	n := 0
	for _, line := range strings.Split(out, "\n") {
		// Subtests are indented; only count top-level ones so a single test
		// with a table of cases cannot inflate the count.
		if strings.HasPrefix(line, "=== RUN   Test") && !strings.Contains(line, "/") {
			n++
		}
	}
	return n
}

// behaviourOnly strips the parts of `go test -v` output that vary between runs
// (timings, goroutine ids, temp paths) so two runs can be compared for equality.
// If a mutant produces output identical to the baseline under this projection,
// the mutation did not change observable behaviour and is therefore invalid.
func behaviourOnly(out string) string {
	var kept []string
	for _, line := range strings.Split(out, "\n") {
		trimmed := strings.TrimSpace(line)
		switch {
		// Keep only the outcome lines. Test NAMES are deliberately dropped:
		// two runs of the same suite can legitimately order or cache tests
		// differently, and keeping the names made every mutant look like it had
		// "changed behaviour" — which defeats the no-op detector entirely.
		case strings.HasPrefix(trimmed, "--- PASS"), strings.HasPrefix(trimmed, "--- FAIL"),
			strings.HasPrefix(trimmed, "--- SKIP"),
			strings.HasPrefix(trimmed, "ok  "), trimmed == "FAIL",
			strings.Contains(trimmed, ".go:"):
			kept = append(kept, trimmed)
		}
	}
	// Counting outcomes is stronger than comparing the sequence: a mutation
	// that swaps which test fails still changes the multiset, and one that
	// changes nothing cannot.
	sort.Strings(kept)
	return strings.Join(kept, "\n")
}

// firstLines trims output for an error message; a failing nested run can be
// thousands of lines and the cause is always near the top.
func firstLines(s string, n int) string {
	lines := strings.Split(s, "\n")
	if len(lines) > n {
		lines = lines[:n]
	}
	return strings.Join(lines, "\n")
}

// TestMutationFilterCoversEveryGuard keeps the filter honest. Dropping a group
// from it silently disarms that area's mutants.
func TestMutationFilterCoversEveryGuard(t *testing.T) {
	for _, group := range []string{
		"ParseModelCatalog",   // nested metadata + non-catalog rejection
		"DiscoveredModelKind", // kind follows upstream metadata
		"Discover",            // URL fallback, auth classification
		"ComposeEndpoint",     // version segment exactly once (chat)
		"ComposeImageEndpoint",
		"ComposeEndpoints", // the composed URLs, both base shapes
		"IsVersionSegment", // "v1beta" vs "vision"
		"DiscoveryResult",  // nil slices are arrays
	} {
		if !strings.Contains(mutationFilter, group) {
			t.Errorf("mutation filter omits %s; mutants there survive silently", group)
		}
	}
}