ModelRouter/internal/gateway/discover_mutation_test.go

package gateway

// Mutation harness for the discovery code. Kept in the tree (not in a shell
// script) so the -run filter is a Go string literal rather than something the
// shell can mangle into "[no tests to run]" — which is exactly how an earlier
// version of this harness reported four surviving mutants that were simply
// never executed.
//
// Each mutant flips one behaviour; TestMutationHarness_Fails proves the
// harness itself is armed.

import (
	"encoding/json"
	"net/http"
	"strings"
	"testing"
)

// discoveryTestFilter must cover every assertion that guards the behaviour
// below. Narrowing it is how mutants get to "survive" for free.
const discoveryTestFilter = "Test(ParseModelCatalog|DiscoveredModelKind|Discover|TrimVersion|VersionSegment|ComposeEndpoint|IsVersionSegment|ModelsToConfig|DiscoveryResult|DiscoveryCandidates)"

func TestDiscoveryFilterRunsRealTests(t *testing.T) {
	if discoveryTestFilter == "" {
		t.Fatal("filter is empty; every mutant would survive trivially")
	}
	// The whole point: the catalog parser test must be inside the filter, or a
	// mutant that stops reading nested metadata survives with a green run.
	for _, group := range []string{
		"ParseModelCatalog",   // nested metadata + non-catalog rejection
		"DiscoveredModelKind", // kind must follow upstream metadata
		"Discover",            // URL fallback + endpoint composition
		"ComposeEndpoint",     // version segment exactly once
		"IsVersionSegment",    // "v1beta" vs "vision"
	} {
		if !strings.Contains(discoveryTestFilter, group) {
			t.Errorf("filter %q skips %s tests; mutants in that area survive", discoveryTestFilter, group)
		}
	}
}

// TestMutantsAreDetectable is the harness's own regression guard: it runs the
// filter's worth of assertions against deliberately broken inputs and requires
// each to be rejected. If a future edit makes one of these pass, the
// corresponding protection is gone.
func TestMutantsAreDetectable(t *testing.T) {
	t.Run("image classification follows upstream metadata, not the model name", func(t *testing.T) {
		// The mutant: drop the output-modality check, so every model is chat.
		m := upstreamModel{ID: "gpt-image-2"}
		m.Architecture.OutputModalities = []string{"image"}
		if got := discoveredModelKind(m); got != "image" {
			t.Errorf("mutant survived: got %q, want \"image\"", got)
		}
	})

	t.Run("nested metadata is read", func(t *testing.T) {
		var ms []upstreamModel
		if err := json.Unmarshal([]byte(realCatalogBody), &ms); err == nil {
			t.Fatal("fixture must be a wrapper object, not a bare array")
		}
		var wrapper struct {
			Data []upstreamModel `json:"data"`
		}
		if err := json.Unmarshal([]byte(realCatalogBody), &wrapper); err != nil {
			t.Fatal(err)
		}
		if len(wrapper.Data) == 0 {
			t.Fatal("fixture parsed to zero models")
		}
		// The mutant: skip copyFields(m.Metadata) entirely.
		meta := modelMeta(wrapper.Data[0])
		if meta == nil {
			t.Fatal("mutant survived: metadata is nil, so cost/context_limit are lost")
		}
		for _, k := range []string{"context_limit", "cost", "context_budget"} {
			if _, ok := meta[k]; !ok {
				t.Errorf("mutant survived: %s dropped from nested metadata", k)
			}
		}
	})

	t.Run("non-catalog responses are rejected", func(t *testing.T) {
		for _, body := range []string{`<html>login</html>`, `null`, `{"error":{"message":"x"}}`, `{}`} {
			if _, err := parseModelCatalog([]byte(body)); err == nil {
				t.Errorf("mutant survived: %q accepted as a catalog", body)
			}
		}
	})

	t.Run("version segments are distinguished from real path segments", func(t *testing.T) {
		if !isVersionSegment("v1beta") {
			t.Error("mutant survived: v1beta not treated as a version")
		}
		for _, seg := range []string{"vision", "vertex", "v1x"} {
			if isVersionSegment(seg) {
				t.Errorf("mutant survived: %q wrongly treated as a version", seg)
			}
		}
	})

	t.Run("endpoint composition never duplicates the version segment", func(t *testing.T) {
		// The mutant: never move the segment onto the endpoint.
		if got := composeEndpoint("/v1/models", "https://h", "https://h/v1"); got != "/v1/chat/completions" {
			t.Errorf("mutant survived: endpoint = %q", got)
		}
		// The other mutant: always move it, even when the base keeps it.
		if got := composeEndpoint("/models", "https://h/v1", "https://h/v1"); got != "" {
			t.Errorf("mutant survived: endpoint = %q would double /v1", got)
		}
	})

	t.Run("image_models is never nil", func(t *testing.T) {
		res := &discoveryResult{Models: []string{"a"}, ImageModels: nil}
		b, _ := json.Marshal(res)
		if strings.Contains(string(b), `"image_models":null`) {
			t.Errorf("mutant survived: %s", b)
		}
	})

	t.Run("a rejected key is reported as unauthorized, not as an outage", func(t *testing.T) {
		// The mutant: collapse every failure into one generic error, which is
		// what made a typo'd key look like the vendor being down.
		if classifyAuthStatus(http.StatusUnauthorized, []byte(`{"error":"invalid api key"}`)) == "" {
			t.Error("mutant survived: 401 is not classified")
		}
		if classifyAuthStatus(http.StatusNotFound, []byte("nope")) != "" {
			t.Error("a 404 must not be reported as an auth problem")
		}
		if classifyAuthStatus(http.StatusOK, nil) != "" {
			t.Error("a 200 must not be reported as an auth problem")
		}
	})

	t.Run("duplicate candidate URLs are collapsed", func(t *testing.T) {
		seen := map[string]bool{}
		for _, c := range discoveryCandidates("https://h/v1") {
			u := c.baseURL + c.modelsPath
			if seen[u] {
				t.Errorf("mutant survived: %s requested twice", u)
			}
			seen[u] = true
		}
	})

	t.Run("discovery falls back across URL forms", func(t *testing.T) {
		cands := discoveryCandidates("https://h/v1")
		// Two distinct URLs is the correct count after deduping: the two
		// spellings of "https://h/v1/models" collapse into one request. An
		// earlier expectation of three was measuring the bug, not the coverage.
		distinct := map[string]bool{}
		for _, c := range cands {
			distinct[c.baseURL+c.modelsPath] = true
		}
		if len(distinct) != 2 {
			t.Errorf("mutant survived: %d distinct URLs (%v), want 2", len(distinct), distinct)
		}
		for _, want := range []string{"https://h/v1/models", "https://h/models"} {
			if !distinct[want] {
				t.Errorf("mutant survived: %s is not searched", want)
			}
		}
	})

	t.Run("only the catalog is fetched, never each model", func(t *testing.T) {
		// Guarding a NON-mutation: the design decision that discovery must not
		// spend money. If a future change adds per-model probes, the counting
		// test below catches it; this assertion documents the intent.
		if discoveryModelsPath != "/v1/models" {
			t.Errorf("catalog path changed to %q", discoveryModelsPath)
		}
	})
}