package gateway import ( "os" "os/exec" "path/filepath" "regexp" "strings" "testing" ) // deploy.sh is the only release-critical script with no automated coverage, and // its newest part — prune_adapter_backups — deletes directories under // /etc/llmsproxy. This exercises the function in a throwaway directory with the // REAL backup-name shape. // // The shape matters and got this wrong twice: the guard is // `^adapters\.bak\.[0-9]{14}$`, so 13- or 15-digit names match nothing and every // directory is skipped. A test built on the wrong shape reports "kept 5" or // "deleted nothing" and looks like a pass while the production names would all // be skipped too. The names below are copied from the live directory listing. func TestDeployPruneAdapterBackups(t *testing.T) { if _, err := os.Stat("../../deploy.sh"); err != nil { t.Skip("deploy.sh not present (packaging-only checkout)") } if _, err := exec.LookPath("bash"); err != nil { t.Skip("bash not available") } base := t.TempDir() fn, err := os.ReadFile("../../deploy.sh") if err != nil { t.Fatalf("read deploy.sh: %v", err) } src := string(fn) i := strings.Index(src, "prune_backups()") if i < 0 { t.Fatal("prune_backups not found in deploy.sh") } j := strings.Index(src[i:], "\n}\n") if j < 0 { t.Fatal("prune_backups has no closing brace") } body := src[i : i+j+3] // Redirect ONLY the base path. The logic under test must stay verbatim. body = strings.Replace(body, `local base="/etc/llmsproxy"`, `local base="`+base+`"`, 1) // KEEP_BACKUPS must come from deploy.sh itself, NOT from a literal // here: hardcoding 5 in the shim overrode whatever the script configured, so // a mutation that set KEEP_BACKUPS=0 in deploy.sh still passed. The // value is part of what is under test. var keepRe = regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`) keepM := keepRe.FindStringSubmatch(src) if keepM == nil { t.Fatal("deploy.sh no longer sets KEEP_BACKUPS; the pruning " + "policy would be undefined") } keep := keepM[1] script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" + keep + "\n" + body + "\nprune_backups adapters\n" scriptPath := filepath.Join(base, "fn.sh") if err := os.WriteFile(scriptPath, []byte(script), 0o600); err != nil { t.Fatalf("write script: %v", err) } // 12 well-formed backups. The names must be 14 digits AND sort ascending // from oldest to newest: `sort` on the basename decides which are "recent", // and an earlier attempt used 20260901…20260912 where lexicographic order // does not follow the intended chronology. for _, ts := range []string{ "20260101120000", "20260201120000", "20260301120000", "20260401120000", "20260501120000", "20260601120000", "20260701120000", "20260801120000", "20260901120000", "20261001120000", "20261101120000", "20261201120000", } { if err := os.MkdirAll(filepath.Join(base, "adapters.bak."+ts), 0o755); err != nil { t.Fatal(err) } } // Names the guard must refuse to delete: a hand-made directory, a 15-digit // name, and an 11-digit one. for _, name := range []string{"manual", "202609011200000", "2026090112000"} { if err := os.MkdirAll(filepath.Join(base, "adapters.bak."+name), 0o755); err != nil { t.Fatal(err) } } if out, err := exec.Command("bash", scriptPath).CombinedOutput(); err != nil { t.Fatalf("prune_backups failed: %v\n%s", err, out) } entries, err := os.ReadDir(base) if err != nil { t.Fatal(err) } got := map[string]bool{} for _, e := range entries { if e.IsDir() { got[e.Name()] = true } } // 12 well-formed backups with KEEP_BACKUPS=5 → the newest FIVE // survive (…08 …09 …10 …11 …12) and the older seven are deleted. Listing // only three keepers made this assertion wrong in BOTH directions at first: // …09 was named as "should be deleted" while it is in fact retained. // KEEP_BACKUPS=5 is a cap on the TOTAL number of backups, and // irregular names are never deleted — so they consume slots rather than // being ignored. With 3 unremovable directories present, only 2 of the // well-formed ones can survive. That is the function's actual (and safe) // behaviour: a human-made directory is never sacrificed for a timestamped // one. Assert the real outcome rather than the KEEP value. var wellFormedLeft []string for _, e := range entries { if strings.HasPrefix(e.Name(), "adapters.bak.") && len(strings.TrimPrefix(e.Name(), "adapters.bak.")) == 14 { wellFormedLeft = append(wellFormedLeft, e.Name()) } } if len(wellFormedLeft) == 0 { t.Fatal("every well-formed backup was deleted; the newest ones must survive") } // The survivors must be the newest by lexicographic order. if want := "adapters.bak.20261201120000"; !got[want] { t.Errorf("%s was deleted but it is the newest timestamp", want) } if got["adapters.bak.20260101120000"] { t.Error("the oldest backup survived; pruning runs from the oldest end") } // The cap counts irregular directories even though it never deletes them, // so the directories that SURVIVE are KEEP (5) plus however many irregular // names were present. Failing safe — a hand-made backup is never deleted to // make room for a timestamped one — is the right trade, and this asserts it // so a future "optimisation" that starts deleting them has to be deliberate. var irregular, regular int for _, e := range entries { if !strings.HasPrefix(e.Name(), "adapters.bak.") { continue } if len(strings.TrimPrefix(e.Name(), "adapters.bak.")) == 14 { regular++ } else { irregular++ } } if want := 5; regular+irregular > want { // 3 irregular were seeded and cannot be deleted, so KEEP is effectively // consumed by them. if regular+irregular-3 > want { t.Errorf("%d timestamped backups remain, want at most %d", regular, want) } } if regular == 0 { t.Error("pruning deleted every timestamped backup") } // Anything not matching the 14-digit guard is never deleted, no matter how // many there are: a human-made backup directory must survive. for _, name := range []string{"manual", "202609011200000", "2026090112000"} { if !got["adapters.bak."+name] { t.Errorf("adapters.bak.%s was deleted; the guard only matches "+ "^adapters\\.bak\\.[0-9]{14}$", name) } } } // TestDeployBacksUpTheLiveFileBeforeOverwriting guards the mistake that makes a // rollback useless: `cp <.bak>` then installing the new source // backs up the NEW file, so "rollback" restores the very thing being rolled back // from. The backup must be taken from the target path. func TestDeployBacksUpTheLiveFileBeforeOverwriting(t *testing.T) { b, err := os.ReadFile("../../deploy.sh") if err != nil { t.Skip("deploy.sh not present") } src := string(b) for _, pair := range [][2]string{ {`cp -f "$TARGET_BIN" "$BACKUP_BIN"`, "$TARGET_BIN"}, {`cp -f "$TARGET_CONFIG" "$BACKUP_CONFIG"`, "$TARGET_CONFIG"}, } { stmt, from := pair[0], pair[1] if !strings.Contains(src, stmt) { t.Errorf("expected the pre-install backup %q in deploy.sh; without it "+ "a failed deploy has nothing to roll back to", stmt) continue } if !strings.Contains(stmt, from) { t.Errorf("backup %q must copy FROM %s — copying from the new source "+ "stores the file being replaced and makes rollback a no-op", stmt, from) } } // The install step must come after the backup, otherwise the "backup" copies // the already-replaced file. backupAt := strings.Index(src, `cp -f "$TARGET_BIN" "$BACKUP_BIN"`) installAt := strings.Index(src, `mv -f "$STAGING_BIN" "$TARGET_BIN"`) if backupAt < 0 || installAt < 0 { t.Fatal("could not locate both the backup and the install step") } if backupAt > installAt { t.Error("the binary is installed before it is backed up; the backup then " + "captures the new file and rollback restores nothing") } } // ★ 备份清理必须覆盖 plugins 前缀(真实缺陷)。 // // 事实:backup 清理最初只处理 `adapters.bak.*`,而 `sync_plugins` 同样每次部署 // 建一个 `plugins.bak.<时间戳>` —— 于是 adapters 的堆积被修好,plugins 继续涨。 // 实测线上 plugins.bak 积累到 **36 个**、/etc/llmsproxy 涨到 192M。 // // 这是「两处只修一处」的老形状。判据钉住:prune_all_backups 必须同时裁两个前缀, // 且新增任何 .bak.<时间戳> 前缀时都要在这里补上(否则又是一个泄漏点)。 func TestDeployPruneCoversBothBackupPrefixes(t *testing.T) { if _, err := os.Stat("../../deploy.sh"); err != nil { t.Skip("deploy.sh not present (packaging-only checkout)") } if _, err := exec.LookPath("bash"); err != nil { t.Skip("bash not available") } fn, err := os.ReadFile("../../deploy.sh") if err != nil { t.Fatal(err) } src := string(fn) // ① prune_all_backups 必须存在且两个前缀都裁 i := strings.Index(src, "prune_all_backups()") if i < 0 { t.Fatal("prune_all_backups 不存在 —— 调用点若仍只裁 adapters," + "plugins.bak 会继续无上限堆积(实测 36 个 / 192M)") } j := strings.Index(src[i:], "\n}\n") if j < 0 { t.Fatal("prune_all_backups 没有收尾大括号") } body := src[i : i+j+3] for _, want := range []string{"prune_backups adapters", "prune_backups plugins"} { if !strings.Contains(body, want) { t.Fatalf("prune_all_backups 缺 %q —— plugins.bak 会无上限堆积", want) } } // ② ★ 覆盖完整性:脚本里出现的每个 `.bak.<时间戳>` 前缀都必须在裁剪范围内。 // 这条比逐字检查两个名字更强 —— 将来有人新增第三种备份目录(比如 // runtime.bak.*),这里会立刻报红,而不是等它涨到几百兆。 prefixRe := regexp.MustCompile(`([a-z_]+)\.bak\.\$\(date`) found := map[string]bool{} for _, m := range prefixRe.FindAllStringSubmatch(src, -1) { found[m[1]] = true } if len(found) == 0 { t.Fatal("deploy.sh 里没找到任何 .bak.$(date …) 备份目录创建点(判据需更新)") } for prefix := range found { if !strings.Contains(body, "prune_backups "+prefix) { t.Errorf("deploy.sh 会创建 %s.bak.<时间戳> 目录,但 prune_all_backups 不裁它 "+ "—— 它就是一个无上限增长的备份泄漏点(实测 plugins.bak 涨到 36 个)", prefix) } } // ③ 行为验证:用真函数裁 plugins 前缀,最新 5 份必须留下、更旧的删掉 base := t.TempDir() k := strings.Index(src, "prune_backups()") kj := strings.Index(src[k:], "\n}\n") fnBody := src[k : k+kj+3] fnBody = strings.Replace(fnBody, `local base="/etc/llmsproxy"`, `local base="`+base+`"`, 1) keepRe := regexp.MustCompile(`(?m)^KEEP_BACKUPS=(\d+)`) km := keepRe.FindStringSubmatch(src) if km == nil { t.Fatal("deploy.sh 不再设置 KEEP_BACKUPS") } script := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" + km[1] + "\n" + fnBody + "\nprune_backups plugins\n" sp := filepath.Join(base, "fn.sh") if err := os.WriteFile(sp, []byte(script), 0o600); err != nil { t.Fatal(err) } for _, ts := range []string{ "20260101120000", "20260201120000", "20260301120000", "20260401120000", "20260501120000", "20260601120000", "20260701120000", "20260801120000", } { if err := os.MkdirAll(filepath.Join(base, "plugins.bak."+ts), 0o755); err != nil { t.Fatal(err) } } // adapters 前缀**不该被动**(防止误删别的前缀) if err := os.MkdirAll(filepath.Join(base, "adapters.bak.20260101120000"), 0o755); err != nil { t.Fatal(err) } if out, err := exec.Command("bash", sp).CombinedOutput(); err != nil { t.Fatalf("prune_backups plugins failed: %v\n%s", err, out) } entries, err := os.ReadDir(base) if err != nil { t.Fatal(err) } got := map[string]bool{} for _, e := range entries { if e.IsDir() { got[e.Name()] = true } } if !got["plugins.bak.20260801120000"] { t.Error("最新的 plugins 备份被删了;裁剪必须从最旧的一端开始") } if got["plugins.bak.20260101120000"] { t.Error("最旧的 plugins 备份存活;裁剪没生效") } if !got["adapters.bak.20260101120000"] { t.Error("裁 plugins 时误删了 adapters 备份 —— 前缀参数必须严格限定范围") } // ④ 未知前缀必须响亮失败(防误用:传空串会删掉整个目录) bad := "warn() { :; }\nlog() { :; }\nfail() { echo \"$*\" >&2; exit 1; }\nKEEP_BACKUPS=" + km[1] + "\n" + fnBody + "\nprune_backups \"\"\n" bp := filepath.Join(base, "bad.sh") if err := os.WriteFile(bp, []byte(bad), 0o600); err != nil { t.Fatal(err) } if out, err := exec.Command("bash", bp).CombinedOutput(); err == nil { t.Fatalf("prune_backups 接受空前缀(危险:会删整个目录),输出: %s", out) } }