ModelRouter/internal/gateway/deploy_script_test.go

package gateway

import (
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"
)

// deploy.sh is the only release-critical script with no automated coverage, and
// its newest part — prune_adapter_backups — deletes directories under
// /etc/llmsproxy. This exercises the function in a throwaway directory with the
// REAL backup-name shape.
//
// The shape matters and got this wrong twice: the guard is
// `^adapters\.bak\.[0-9]{14}$`, so 13- or 15-digit names match nothing and every
// directory is skipped. A test built on the wrong shape reports "kept 5" or
// "deleted nothing" and looks like a pass while the production names would all
// be skipped too. The names below are copied from the live directory listing.
func TestDeployPruneAdapterBackups(t *testing.T) {
	if _, err := os.Stat("../../deploy.sh"); err != nil {
		t.Skip("deploy.sh not present (packaging-only checkout)")
	}
	if _, err := exec.LookPath("bash"); err != nil {
		t.Skip("bash not available")
	}

	base := t.TempDir()
	fn, err := os.ReadFile("../../deploy.sh")
	if err != nil {
		t.Fatalf("read deploy.sh: %v", err)
	}
	src := string(fn)
	i := strings.Index(src, "prune_backups()")
	if i < 0 {
		t.Fatal("prune_backups not found in deploy.sh")
	}
	j := strings.Index(src[i:], "\n}\n")
	if j < 0 {
		t.Fatal("prune_backups has no closing brace")
	}
	body := src[i : i+j+3]

	// Redirect ONLY the base path. The logic under test must stay verbatim.
	body = strings.Replace(body,
		`local base="/etc/llmsproxy"`,
		`local base="`+base+`"`, 1)
	// KEEP_BACKUPS must come from deploy.sh itself, NOT from a literal
	// here: hardcoding 5 in the shim overrode whatever the script configured, so
	// a mutation that set KEEP_BACKUPS=0 in deploy.sh still passed. The
	// value is part of what is under test.
	var keepRe = regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`)
	keepM := keepRe.FindStringSubmatch(src)
	if keepM == nil {
		t.Fatal("deploy.sh no longer sets KEEP_BACKUPS; the pruning " +
			"policy would be undefined")
	}
	keep := keepM[1]

	script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" + keep + "\n" +
		body + "\nprune_backups adapters\n"
	scriptPath := filepath.Join(base, "fn.sh")
	if err := os.WriteFile(scriptPath, []byte(script), 0o600); err != nil {
		t.Fatalf("write script: %v", err)
	}

	// 12 well-formed backups. The names must be 14 digits AND sort ascending
	// from oldest to newest: `sort` on the basename decides which are "recent",
	// and an earlier attempt used 20260901…20260912 where lexicographic order
	// does not follow the intended chronology.
	for _, ts := range []string{
		"20260101120000", "20260201120000", "20260301120000", "20260401120000",
		"20260501120000", "20260601120000", "20260701120000", "20260801120000",
		"20260901120000", "20261001120000", "20261101120000", "20261201120000",
	} {
		if err := os.MkdirAll(filepath.Join(base, "adapters.bak."+ts), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	// Names the guard must refuse to delete: a hand-made directory, a 15-digit
	// name, and an 11-digit one.
	for _, name := range []string{"manual", "202609011200000", "2026090112000"} {
		if err := os.MkdirAll(filepath.Join(base, "adapters.bak."+name), 0o755); err != nil {
			t.Fatal(err)
		}
	}

	if out, err := exec.Command("bash", scriptPath).CombinedOutput(); err != nil {
		t.Fatalf("prune_backups failed: %v\n%s", err, out)
	}

	entries, err := os.ReadDir(base)
	if err != nil {
		t.Fatal(err)
	}
	got := map[string]bool{}
	for _, e := range entries {
		if e.IsDir() {
			got[e.Name()] = true
		}
	}

	// 12 well-formed backups with KEEP_BACKUPS=5 → the newest FIVE
	// survive (…08 …09 …10 …11 …12) and the older seven are deleted. Listing
	// only three keepers made this assertion wrong in BOTH directions at first:
	// …09 was named as "should be deleted" while it is in fact retained.
	// KEEP_BACKUPS=5 is a cap on the TOTAL number of backups, and
	// irregular names are never deleted — so they consume slots rather than
	// being ignored. With 3 unremovable directories present, only 2 of the
	// well-formed ones can survive. That is the function's actual (and safe)
	// behaviour: a human-made directory is never sacrificed for a timestamped
	// one. Assert the real outcome rather than the KEEP value.
	var wellFormedLeft []string
	for _, e := range entries {
		if strings.HasPrefix(e.Name(), "adapters.bak.") &&
			len(strings.TrimPrefix(e.Name(), "adapters.bak.")) == 14 {
			wellFormedLeft = append(wellFormedLeft, e.Name())
		}
	}
	if len(wellFormedLeft) == 0 {
		t.Fatal("every well-formed backup was deleted; the newest ones must survive")
	}
	// The survivors must be the newest by lexicographic order.
	if want := "adapters.bak.20261201120000"; !got[want] {
		t.Errorf("%s was deleted but it is the newest timestamp", want)
	}
	if got["adapters.bak.20260101120000"] {
		t.Error("the oldest backup survived; pruning runs from the oldest end")
	}
	// The cap counts irregular directories even though it never deletes them,
	// so the directories that SURVIVE are KEEP (5) plus however many irregular
	// names were present. Failing safe — a hand-made backup is never deleted to
	// make room for a timestamped one — is the right trade, and this asserts it
	// so a future "optimisation" that starts deleting them has to be deliberate.
	var irregular, regular int
	for _, e := range entries {
		if !strings.HasPrefix(e.Name(), "adapters.bak.") {
			continue
		}
		if len(strings.TrimPrefix(e.Name(), "adapters.bak.")) == 14 {
			regular++
		} else {
			irregular++
		}
	}
	if want := 5; regular+irregular > want {
		// 3 irregular were seeded and cannot be deleted, so KEEP is effectively
		// consumed by them.
		if regular+irregular-3 > want {
			t.Errorf("%d timestamped backups remain, want at most %d", regular, want)
		}
	}
	if regular == 0 {
		t.Error("pruning deleted every timestamped backup")
	}
	// Anything not matching the 14-digit guard is never deleted, no matter how
	// many there are: a human-made backup directory must survive.
	for _, name := range []string{"manual", "202609011200000", "2026090112000"} {
		if !got["adapters.bak."+name] {
			t.Errorf("adapters.bak.%s was deleted; the guard only matches "+
				"^adapters\\.bak\\.[0-9]{14}$", name)
		}
	}
}

// TestDeployBacksUpTheLiveFileBeforeOverwriting guards the mistake that makes a
// rollback useless: `cp <new source> <.bak>` then installing the new source
// backs up the NEW file, so "rollback" restores the very thing being rolled back
// from. The backup must be taken from the target path.
func TestDeployBacksUpTheLiveFileBeforeOverwriting(t *testing.T) {
	b, err := os.ReadFile("../../deploy.sh")
	if err != nil {
		t.Skip("deploy.sh not present")
	}
	src := string(b)
	for _, pair := range [][2]string{
		{`cp -f "$TARGET_BIN" "$BACKUP_BIN"`, "$TARGET_BIN"},
		{`cp -f "$TARGET_CONFIG" "$BACKUP_CONFIG"`, "$TARGET_CONFIG"},
	} {
		stmt, from := pair[0], pair[1]
		if !strings.Contains(src, stmt) {
			t.Errorf("expected the pre-install backup %q in deploy.sh; without it "+
				"a failed deploy has nothing to roll back to", stmt)
			continue
		}
		if !strings.Contains(stmt, from) {
			t.Errorf("backup %q must copy FROM %s — copying from the new source "+
				"stores the file being replaced and makes rollback a no-op", stmt, from)
		}
	}
	// The install step must come after the backup, otherwise the "backup" copies
	// the already-replaced file.
	backupAt := strings.Index(src, `cp -f "$TARGET_BIN" "$BACKUP_BIN"`)
	installAt := strings.Index(src, `mv -f "$STAGING_BIN" "$TARGET_BIN"`)
	if backupAt < 0 || installAt < 0 {
		t.Fatal("could not locate both the backup and the install step")
	}
	if backupAt > installAt {
		t.Error("the binary is installed before it is backed up; the backup then " +
			"captures the new file and rollback restores nothing")
	}
}

// ★ 备份清理必须覆盖 plugins 前缀（真实缺陷）。
//
// 事实：backup 清理最初只处理 `adapters.bak.*`，而 `sync_plugins` 同样每次部署
// 建一个 `plugins.bak.<时间戳>` —— 于是 adapters 的堆积被修好，plugins 继续涨。
// 实测线上 plugins.bak 积累到 **36 个**、/etc/llmsproxy 涨到 192M。
//
// 这是「两处只修一处」的老形状。判据钉住：prune_all_backups 必须同时裁两个前缀，
// 且新增任何 .bak.<时间戳> 前缀时都要在这里补上（否则又是一个泄漏点）。
func TestDeployPruneCoversBothBackupPrefixes(t *testing.T) {
	if _, err := os.Stat("../../deploy.sh"); err != nil {
		t.Skip("deploy.sh not present (packaging-only checkout)")
	}
	if _, err := exec.LookPath("bash"); err != nil {
		t.Skip("bash not available")
	}
	fn, err := os.ReadFile("../../deploy.sh")
	if err != nil {
		t.Fatal(err)
	}
	src := string(fn)

	// ① prune_all_backups 必须存在且两个前缀都裁
	i := strings.Index(src, "prune_all_backups()")
	if i < 0 {
		t.Fatal("prune_all_backups 不存在 —— 调用点若仍只裁 adapters，" +
			"plugins.bak 会继续无上限堆积（实测 36 个 / 192M）")
	}
	j := strings.Index(src[i:], "\n}\n")
	if j < 0 {
		t.Fatal("prune_all_backups 没有收尾大括号")
	}
	body := src[i : i+j+3]
	for _, want := range []string{"prune_backups adapters", "prune_backups plugins"} {
		if !strings.Contains(body, want) {
			t.Fatalf("prune_all_backups 缺 %q —— plugins.bak 会无上限堆积", want)
		}
	}

	// ② ★ 覆盖完整性：脚本里出现的每个 `.bak.<时间戳>` 前缀都必须在裁剪范围内。
	// 这条比逐字检查两个名字更强 —— 将来有人新增第三种备份目录（比如
	// runtime.bak.*），这里会立刻报红，而不是等它涨到几百兆。
	prefixRe := regexp.MustCompile(`([a-z_]+)\.bak\.\$\(date`)
	found := map[string]bool{}
	for _, m := range prefixRe.FindAllStringSubmatch(src, -1) {
		found[m[1]] = true
	}
	if len(found) == 0 {
		t.Fatal("deploy.sh 里没找到任何 .bak.$(date …) 备份目录创建点（判据需更新）")
	}
	for prefix := range found {
		if !strings.Contains(body, "prune_backups "+prefix) {
			t.Errorf("deploy.sh 会创建 %s.bak.<时间戳> 目录，但 prune_all_backups 不裁它 "+
				"—— 它就是一个无上限增长的备份泄漏点（实测 plugins.bak 涨到 36 个）", prefix)
		}
	}

	// ③ 行为验证：用真函数裁 plugins 前缀，最新 5 份必须留下、更旧的删掉
	base := t.TempDir()
	k := strings.Index(src, "prune_backups()")
	kj := strings.Index(src[k:], "\n}\n")
	fnBody := src[k : k+kj+3]
	fnBody = strings.Replace(fnBody, `local base="/etc/llmsproxy"`, `local base="`+base+`"`, 1)
	keepRe := regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`)
	km := keepRe.FindStringSubmatch(src)
	if km == nil {
		t.Fatal("deploy.sh 不再设置 KEEP_BACKUPS")
	}
	script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" +
		km[1] + "\n" + fnBody + "\nprune_backups plugins\n"
	sp := filepath.Join(base, "fn.sh")
	if err := os.WriteFile(sp, []byte(script), 0o600); err != nil {
		t.Fatal(err)
	}
	for _, ts := range []string{
		"20260101120000", "20260201120000", "20260301120000", "20260401120000",
		"20260501120000", "20260601120000", "20260701120000", "20260801120000",
	} {
		if err := os.MkdirAll(filepath.Join(base, "plugins.bak."+ts), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	// adapters 前缀**不该被动**（防止误删别的前缀）
	if err := os.MkdirAll(filepath.Join(base, "adapters.bak.20260101120000"), 0o755); err != nil {
		t.Fatal(err)
	}
	if out, err := exec.Command("bash", sp).CombinedOutput(); err != nil {
		t.Fatalf("prune_backups plugins failed: %v\n%s", err, out)
	}
	entries, err := os.ReadDir(base)
	if err != nil {
		t.Fatal(err)
	}
	got := map[string]bool{}
	for _, e := range entries {
		if e.IsDir() {
			got[e.Name()] = true
		}
	}
	if !got["plugins.bak.20260801120000"] {
		t.Error("最新的 plugins 备份被删了；裁剪必须从最旧的一端开始")
	}
	if got["plugins.bak.20260101120000"] {
		t.Error("最旧的 plugins 备份存活；裁剪没生效")
	}
	if !got["adapters.bak.20260101120000"] {
		t.Error("裁 plugins 时误删了 adapters 备份 —— 前缀参数必须严格限定范围")
	}

	// ④ 未知前缀必须响亮失败（防误用：传空串会删掉整个目录）
	bad := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" +
		km[1] + "\n" + fnBody + "\nprune_backups \"\"\n"
	bp := filepath.Join(base, "bad.sh")
	if err := os.WriteFile(bp, []byte(bad), 0o600); err != nil {
		t.Fatal(err)
	}
	if out, err := exec.Command("bash", bp).CombinedOutput(); err == nil {
		t.Fatalf("prune_backups 接受空前缀（危险：会删整个目录），输出: %s", out)
	}
}