三层回退恢复机制(L0写前留档/L1恢复梯子/L2离线回滚)+ guard 父守护

- L0: files 插件写受保护系统路径(/etc 等)前自动留档,AbstractBeforeWrite 到 data/file_baseline
- L1: failback 受限 worker 执行恢复梯子 probe→还原DNS/proxy→还原LLM配置+ReloadFromConfig→probe,N轮有界
- L2: tracker changeset 持久化原文 blob,guard 离线 RollbackFromDisk 回滚 agentfs;SystemSnapshot 支撑
- guard 父守护: 心跳 IPC(PING/ACK unix socket, 文件心跳回退)、失败计数、退出码协议(42/43/44)、最后手段
- 发行版路径适配: system.protected_paths/network_paths 可注入,默认面向主流 Linux
- Windows 兼容: guard.go/failback.go 加 //go:build linux, guard_windows.go 提供 no-op 桩
- 修复: guard.yaml last_resort 键冲突、changeset Content 不落盘导致离线回滚丢原文

Build 全绿, vet 干净, system/recovery/ipc/tracker 单元测试全过
This commit is contained in:
root
2026-08-05 16:00:08 +08:00
parent 33e97f2a99
commit bc9ef15eb0
28 changed files with 3675 additions and 167 deletions

View File

@ -40,6 +40,9 @@ func fileHash(path string) (string, int64, error) {
return hex.EncodeToString(h[:]), int64(len(data)), nil
}
// maxCapturedContent 回滚内容捕获上限:超大文件不保存原文(回滚时跳过并告警)。
const maxCapturedContent = 8 << 20
func fileInfo(path string) (size int64, modTime time.Time, err error) {
info, err := os.Stat(path)
if err != nil {
@ -53,6 +56,7 @@ type FSState struct {
Root string `json:"root"`
}
// captureFSState 仅记录哈希/尺寸(用于"之后"快照,省内存)。
func captureFSState(root string) (*FSState, error) {
state := &FSState{
Files: make(map[string]FileChange),
@ -77,6 +81,27 @@ func captureFSState(root string) (*FSState, error) {
return state, err
}
// captureFSStateWithContent 额外捕获文件原文(用于"之前"基线,供回滚还原被改/被删文件)。
func captureFSStateWithContent(root string) (*FSState, error) {
state, err := captureFSState(root)
if err != nil {
return nil, err
}
for rel := range state.Files {
path := filepath.Join(root, rel)
info, err := os.Stat(path)
if err != nil || info.Size() > maxCapturedContent {
continue
}
if data, err := os.ReadFile(path); err == nil {
fc := state.Files[rel]
fc.Content = data
state.Files[rel] = fc
}
}
return state, nil
}
func diffStates(before, after *FSState) []FileChange {
var changes []FileChange
if before == nil || after == nil {
@ -95,6 +120,7 @@ func diffStates(before, after *FSState) []FileChange {
HashAfter: afterFile.HashAfter,
SizeBefore: beforeFile.SizeAfter,
SizeAfter: afterFile.SizeAfter,
Content: beforeFile.Content, // 原始内容,供回滚还原
})
}
} else {
@ -114,6 +140,7 @@ func diffStates(before, after *FSState) []FileChange {
Type: ChangeFileDeleted,
HashBefore: before.Files[path].HashAfter,
SizeBefore: before.Files[path].SizeAfter,
Content: before.Files[path].Content, // 原始内容,供回滚还原
})
}
}

114
internal/tracker/disk.go Normal file
View File

@ -0,0 +1,114 @@
package tracker
import (
"encoding/json"
"fmt"
"log"
"os"
"path/filepath"
"sort"
"strings"
"time"
)
// LoadChangeSetsFromDisk 把 <dataDir>/changesets/*.json 读回内存 changeSets
// (按修改时间正序),使 guard 可在 worker 未运行时离线回滚 agentfs。
func (t *Tracker) LoadChangeSetsFromDisk() (int, error) {
dir := filepath.Join(t.dataDir, "changesets")
entries, err := os.ReadDir(dir)
if err != nil {
if os.IsNotExist(err) {
return 0, nil
}
return 0, err
}
type csFile struct {
path string
mod time.Time
}
var files []csFile
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
continue
}
info, err := e.Info()
if err != nil {
continue
}
files = append(files, csFile{path: filepath.Join(dir, e.Name()), mod: info.ModTime()})
}
sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
t.mu.Lock()
defer t.mu.Unlock()
t.changeSets = t.changeSets[:0]
for _, f := range files {
data, err := os.ReadFile(f.path)
if err != nil {
continue
}
var cs ChangeSet
if err := json.Unmarshal(data, &cs); err != nil {
continue
}
t.loadContentBlobs(&cs, dir)
t.changeSets = append(t.changeSets, &cs)
}
log.Printf("[tracker] loaded %d changesets from disk", len(t.changeSets))
return len(t.changeSets), nil
}
// RollbackFromDisk 供 guard 在 worker 离线时执行 L2 agentfs 回滚:
// 读回全部持久化 changeset 并按时间逆序逆应用(还原被改/被删文件、删除新增),
// 然后删除这些 changeset 文件。返回还原的 changeset 数。
func (t *Tracker) RollbackFromDisk() (int, error) {
if _, err := t.LoadChangeSetsFromDisk(); err != nil {
return 0, err
}
t.mu.Lock()
defer t.mu.Unlock()
n := len(t.changeSets)
if n == 0 {
log.Printf("[tracker] rollback from disk: nothing to revert")
return 0, nil
}
for i := n - 1; i >= 0; i-- {
t.applyReverseLocked(t.changeSets[i])
}
dir := filepath.Join(t.dataDir, "changesets")
for _, cs := range t.changeSets {
os.Remove(filepath.Join(dir, cs.ID+".json"))
removeContentBlobs(dir, cs.ID)
}
t.changeSets = t.changeSets[:0]
log.Printf("[tracker] rollback from disk: reverted %d change sets", n)
return n, nil
}
// ChangesetsOnDisk 返回磁盘上持久化 changeset 数量(guard 决策用)。
func (t *Tracker) ChangesetsOnDisk() int {
dir := filepath.Join(t.dataDir, "changesets")
entries, err := os.ReadDir(dir)
if err != nil {
return 0
}
n := 0
for _, e := range entries {
if !e.IsDir() && strings.HasSuffix(e.Name(), ".json") {
n++
}
}
return n
}
// NewOfflineTracker 构造一个仅用于离线回滚的 tracker(不 mount overlay)。
// worker 目录不存在时也会自动创建(Init 语义)。
func NewOfflineTracker(dataDir, workDir string) *Tracker {
t := NewTracker(dataDir, workDir)
_ = t.Init()
return t
}
var _ = fmt.Sprintf

View File

@ -0,0 +1,92 @@
package tracker
import (
"os"
"path/filepath"
"testing"
)
// 构造一条写入磁盘的 changeset(模拟 worker 运行期间的变更),随后用
// RollbackFromDisk 离线还原。
func TestRollbackFromDisk(t *testing.T) {
dataDir := t.TempDir()
workDir := t.TempDir()
trk := NewTracker(dataDir, workDir)
if err := trk.Init(); err != nil {
t.Fatalf("Init: %v", err)
}
upper := trk.upperDir
os.MkdirAll(upper, 0755)
// 1. 先创建一个文件(对应"新建")
newFile := filepath.Join(upper, "new.txt")
os.WriteFile(newFile, []byte("brand new"), 0644)
// 2. 修改一个文件(对应"修改",带原文)
modFile := filepath.Join(upper, "mod.txt")
os.WriteFile(modFile, []byte("after"), 0644)
beforeContent := []byte("before")
csMod := &ChangeSet{
ID: "cs_mod",
Files: []FileChange{{
Path: "mod.txt",
Type: ChangeFileModified,
Content: beforeContent,
}},
}
trk.saveChangeSet(csMod)
csNew := &ChangeSet{
ID: "cs_new",
Files: []FileChange{{
Path: "new.txt",
Type: ChangeFileCreated,
}},
}
trk.saveChangeSet(csNew)
if trk.ChangesetsOnDisk() != 2 {
t.Fatalf("ChangesetsOnDisk = %d, want 2", trk.ChangesetsOnDisk())
}
// 离线回滚(模拟 guard 在 worker 崩溃后调用)
n, err := trk.RollbackFromDisk()
if err != nil {
t.Fatalf("RollbackFromDisk: %v", err)
}
if n != 2 {
t.Fatalf("rolled back %d, want 2", n)
}
if trk.ChangesetsOnDisk() != 0 {
t.Fatalf("ChangesetsOnDisk after rollback = %d, want 0", trk.ChangesetsOnDisk())
}
// 新建文件被删除
if _, err := os.Stat(newFile); !os.IsNotExist(err) {
t.Fatal("created file should be removed after rollback")
}
// 修改文件还原原文
data, err := os.ReadFile(modFile)
if err != nil {
t.Fatalf("read modFile: %v", err)
}
if string(data) != "before" {
t.Fatalf("modFile restored to %q, want %q", data, beforeContent)
}
}
func TestRollbackFromDiskEmpty(t *testing.T) {
dataDir := t.TempDir()
workDir := t.TempDir()
trk := NewTracker(dataDir, workDir)
if err := trk.Init(); err != nil {
t.Fatalf("Init: %v", err)
}
n, err := trk.RollbackFromDisk()
if err != nil {
t.Fatalf("RollbackFromDisk: %v", err)
}
if n != 0 {
t.Fatalf("rolled back %d, want 0", n)
}
}

View File

@ -26,7 +26,7 @@ type Tracker struct {
active bool
before *FSState
changeSets []*ChangeSet
keepChangesets int // 保留最近 N 份 changeset,0 = 不限
keepChangesets int // 保留最近 N 份 changeset,0 = 不限
maxChangesetAge time.Duration // changeset 最大保留时长,0 = 不限
}
@ -115,7 +115,7 @@ func (t *Tracker) PostAction(action string) *ChangeSet {
t.mu.Lock()
defer t.mu.Unlock()
after := t.capture()
after, _ := captureFSState(t.upperDir)
changes := diffStates(t.before, after)
cs := NewChangeSet(action)
@ -146,7 +146,7 @@ func (t *Tracker) ChangeSets() []*ChangeSet {
}
func (t *Tracker) capture() *FSState {
state, err := captureFSState(t.upperDir)
state, err := captureFSStateWithContent(t.upperDir)
if err != nil {
return &FSState{Files: make(map[string]FileChange), Root: t.upperDir}
}
@ -192,6 +192,41 @@ func (t *Tracker) saveChangeSet(cs *ChangeSet) {
if err := os.WriteFile(path, data, 0644); err != nil {
log.Printf("[tracker] write changeset %s: %v", cs.ID, err)
}
// Content 字段是 json:"-",不随 JSON 落盘;为保证 L2 离线回滚(guard 重启后)
// 仍能还原被改/被删文件,把回滚原文作为伴随 blob 单独持久化。
t.saveContentBlobs(cs, dir)
}
// saveContentBlobs 把 changeset 中各文件的回滚原文写入 <dataDir>/changesets/<csID>_blobs/<i>.bin。
func (t *Tracker) saveContentBlobs(cs *ChangeSet, dir string) {
blobDir := filepath.Join(dir, cs.ID+"_blobs")
os.MkdirAll(blobDir, 0755)
for i, f := range cs.Files {
if len(f.Content) == 0 {
continue
}
bp := filepath.Join(blobDir, fmt.Sprintf("%03d.bin", i))
if err := os.WriteFile(bp, f.Content, 0600); err != nil {
log.Printf("[tracker] write blob %s: %v", bp, err)
}
}
}
// loadContentBlobs 读回 <csID>_blobs 目录中的回滚原文到 FileChange.Content。
func (t *Tracker) loadContentBlobs(cs *ChangeSet, dir string) {
blobDir := filepath.Join(dir, cs.ID+"_blobs")
for i := range cs.Files {
bp := filepath.Join(blobDir, fmt.Sprintf("%03d.bin", i))
if data, err := os.ReadFile(bp); err == nil {
cs.Files[i].Content = data
}
}
}
// removeContentBlobs 删除一个 changeset 的 blob 目录。
func removeContentBlobs(dir, id string) {
os.RemoveAll(filepath.Join(dir, id+"_blobs"))
}
func (t *Tracker) cleanupChangeSets() {
@ -202,8 +237,8 @@ func (t *Tracker) cleanupChangeSets() {
}
type csFile struct {
name string
info os.FileInfo
name string
info os.FileInfo
}
var files []csFile
for _, e := range entries {
@ -243,6 +278,7 @@ func (t *Tracker) cleanupChangeSets() {
for i := 0; i < excess; i++ {
path := filepath.Join(dir, remaining[i].name)
os.Remove(path)
removeContentBlobs(dir, strings.TrimSuffix(remaining[i].name, ".json"))
}
remaining = remaining[excess:]
}
@ -253,6 +289,9 @@ func (t *Tracker) cleanupChangeSets() {
}
}
// Rollback 全量回滚:按时间逆序应用每个 changeset 的逆操作,把工作区恢复到
// 首条 changeset 之前的状态(用捕获的原文还原被改/被删文件、删除新增文件)。
// 无任何 changeset 时退回整目录重置(移除 upper 重建,丢弃全部变更)。
func (t *Tracker) Rollback() error {
t.mu.Lock()
defer t.mu.Unlock()
@ -263,22 +302,77 @@ func (t *Tracker) Rollback() error {
}
}
if len(t.changeSets) > 0 {
for i := len(t.changeSets) - 1; i >= 0; i-- {
t.applyReverseLocked(t.changeSets[i])
}
log.Printf("[tracker] rollback complete: reverted %d change sets", len(t.changeSets))
} else {
if err := t.resetUpperLocked(); err != nil {
return err
}
log.Printf("[tracker] rollback complete: no change sets, reset upper dir")
}
t.changeSets = nil
t.before = nil
t.mounted = false
return nil
}
// RollbackLatest 仅撤销最近一条 changeset(定向回滚,不动更早的改动)。
func (t *Tracker) RollbackLatest() error {
t.mu.Lock()
defer t.mu.Unlock()
if len(t.changeSets) == 0 {
return fmt.Errorf("no change sets to roll back")
}
cs := t.changeSets[len(t.changeSets)-1]
t.applyReverseLocked(cs)
t.changeSets = t.changeSets[:len(t.changeSets)-1]
t.before = t.capture()
log.Printf("[tracker] rolled back latest change set %s (%d files)", cs.ID, len(cs.Files))
return nil
}
// applyReverseLocked 逆应用一个 changeset:created→删除;modified→写回原文;deleted→用原文重建。
// 调用方须持有写锁。
func (t *Tracker) applyReverseLocked(cs *ChangeSet) {
for _, f := range cs.Files {
path := filepath.Join(t.upperDir, filepath.Clean(f.Path))
switch f.Type {
case ChangeFileCreated:
if err := os.RemoveAll(path); err != nil {
log.Printf("[tracker] rollback remove %s: %v", f.Path, err)
}
case ChangeFileModified, ChangeFileDeleted:
if len(f.Content) == 0 {
log.Printf("[tracker] rollback %s: original content not captured, skipping", f.Path)
continue
}
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
log.Printf("[tracker] rollback mkdir %s: %v", filepath.Dir(f.Path), err)
continue
}
if err := os.WriteFile(path, f.Content, 0644); err != nil {
log.Printf("[tracker] rollback restore %s: %v", f.Path, err)
}
}
}
}
// resetUpperLocked 整目录重置(无 changeset 时的兜底),调用方须持有写锁。
func (t *Tracker) resetUpperLocked() error {
if err := os.RemoveAll(t.upperDir); err != nil {
return fmt.Errorf("remove upper: %w", err)
}
if err := os.RemoveAll(filepath.Join(t.workDir, "work")); err != nil {
return fmt.Errorf("remove work: %w", err)
}
if err := os.MkdirAll(t.upperDir, 0755); err != nil {
return fmt.Errorf("recreate upper: %w", err)
}
t.changeSets = nil
t.before = nil
t.mounted = false
log.Printf("[tracker] rollback complete")
return nil
}

View File

@ -312,3 +312,134 @@ func TestCaptureDirNotExist(t *testing.T) {
t.Error("expected error for nonexistent directory")
}
}
func TestDiffStatesModifiedContent(t *testing.T) {
dir := t.TempDir()
f := filepath.Join(dir, "f.txt")
os.WriteFile(f, []byte("original-content"), 0644)
before, err := captureFSStateWithContent(dir)
if err != nil {
t.Fatal(err)
}
os.WriteFile(f, []byte("tampered-content"), 0644)
after, _ := captureFSState(dir)
changes := diffStates(before, after)
if len(changes) != 1 || changes[0].Type != ChangeFileModified {
t.Fatalf("expected 1 modified, got %+v", changes)
}
if string(changes[0].Content) != "original-content" {
t.Fatalf("modified change should carry original content, got %q", changes[0].Content)
}
}
func TestDiffStatesDeletedContent(t *testing.T) {
dir := t.TempDir()
f := filepath.Join(dir, "f.txt")
os.WriteFile(f, []byte("do-not-lose"), 0644)
before, err := captureFSStateWithContent(dir)
if err != nil {
t.Fatal(err)
}
os.Remove(f)
after, _ := captureFSState(dir)
changes := diffStates(before, after)
if len(changes) != 1 || changes[0].Type != ChangeFileDeleted {
t.Fatalf("expected 1 deleted, got %+v", changes)
}
if string(changes[0].Content) != "do-not-lose" {
t.Fatalf("deleted change should carry original content, got %q", changes[0].Content)
}
}
func TestRollbackLatest(t *testing.T) {
dir := t.TempDir()
work := filepath.Join(dir, "work")
tr := NewTracker(filepath.Join(dir, "data"), work)
if err := tr.Init(); err != nil {
t.Fatal(err)
}
upper := tr.upperDir
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("stable"), 0644)
os.WriteFile(filepath.Join(upper, "gone.txt"), []byte("do-not-lose"), 0644)
// 动作1:改 keep、加 new(应保留)
tr.PreAction("action1")
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("tampered"), 0644)
os.WriteFile(filepath.Join(upper, "new.txt"), []byte("added"), 0644)
tr.PostAction("action1")
// 动作2:删 gone(仅撤销这条)
tr.PreAction("action2")
os.Remove(filepath.Join(upper, "gone.txt"))
tr.PostAction("action2")
if !tr.HasChanges() {
t.Fatal("expected changes after PostAction")
}
if err := tr.RollbackLatest(); err != nil {
t.Fatalf("RollbackLatest: %v", err)
}
restored, err := os.ReadFile(filepath.Join(upper, "gone.txt"))
if err != nil || string(restored) != "do-not-lose" {
t.Fatalf("gone.txt should be recreated with original content, got %q err=%v", restored, err)
}
// action1 的改动不受影响
got, err := os.ReadFile(filepath.Join(upper, "keep.txt"))
if err != nil || string(got) != "tampered" {
t.Fatalf("keep.txt should keep action1 changes, got %q err=%v", got, err)
}
if _, err := os.Stat(filepath.Join(upper, "new.txt")); err != nil {
t.Fatalf("new.txt should still exist after latest-only rollback, err=%v", err)
}
if tr.HasChanges() != true {
t.Fatal("earlier change sets should remain after latest-only rollback")
}
if got := len(tr.ChangeSets()); got != 1 {
t.Fatalf("expected 1 remaining change set, got %d", got)
}
}
func TestRollbackFull(t *testing.T) {
dir := t.TempDir()
work := filepath.Join(dir, "work")
tr := NewTracker(filepath.Join(dir, "data"), work)
if err := tr.Init(); err != nil {
t.Fatal(err)
}
upper := tr.upperDir
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("stable"), 0644)
os.WriteFile(filepath.Join(upper, "deleteme.txt"), []byte("bye"), 0644)
// 动作1:改 keep、加 new
tr.PreAction("a")
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("tampered"), 0644)
os.WriteFile(filepath.Join(upper, "new.txt"), []byte("added"), 0644)
tr.PostAction("a")
// 动作2:删 deleteme(在受追踪的动作内)
tr.PreAction("b")
os.Remove(filepath.Join(upper, "deleteme.txt"))
tr.PostAction("b")
if err := tr.Rollback(); err != nil {
t.Fatalf("Rollback: %v", err)
}
got, _ := os.ReadFile(filepath.Join(upper, "keep.txt"))
if string(got) != "stable" {
t.Fatalf("keep.txt should be restored, got %q", got)
}
if _, err := os.Stat(filepath.Join(upper, "new.txt")); !os.IsNotExist(err) {
t.Fatalf("new.txt should be gone after full rollback")
}
restored, err := os.ReadFile(filepath.Join(upper, "deleteme.txt"))
if err != nil || string(restored) != "bye" {
t.Fatalf("deleted file should be recreated with original content, got %q err=%v", restored, err)
}
}