三层回退恢复机制（L0写前留档/L1恢复梯子/L2离线回滚）+ guard 父守护

- L0: files 插件写受保护系统路径(/etc 等)前自动留档，AbstractBeforeWrite 到 data/file_baseline - L1: failback 受限 worker 执行恢复梯子 probe→还原DNS/proxy→还原LLM配置+ReloadFromConfig→probe，N轮有界 - L2: tracker changeset 持久化原文 blob，guard 离线 RollbackFromDisk 回滚 agentfs；SystemSnapshot 支撑 - guard 父守护: 心跳 IPC(PING/ACK unix socket, 文件心跳回退)、失败计数、退出码协议(42/43/44)、最后手段 - 发行版路径适配: system.protected_paths/network_paths 可注入，默认面向主流 Linux - Windows 兼容: guard.go/failback.go 加 //go:build linux, guard_windows.go 提供 no-op 桩 - 修复: guard.yaml last_resort 键冲突、changeset Content 不落盘导致离线回滚丢原文 Build 全绿, vet 干净, system/recovery/ipc/tracker 单元测试全过
2026-10-03 15:53:56 +00:00 · 2026-08-05 16:00:08 +08:00
parent 33e97f2a99
commit bc9ef15eb0
28 changed files with 3675 additions and 167 deletions
--- a/cmd/homed/failback.go
+++ b/cmd/homed/failback.go
@ -0,0 +1,138 @@
+//go:build linux
+
+// L1 failback 恢复 worker（Linux 专属）：在受限 boot 下执行恢复梯子，
+// 依赖 DNS/代理网络基线、LLM 配置快照与 reload 等 Linux 语义。
+// 非 Linux 平台不编译，由 guard_windows.go 提供桩。
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"time"
+
+	agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
+	internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
+	luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
+	"gitcode.com/JianFeeeee/HomeAgent/internal/recovery"
+	sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
+	"gitcode.com/JianFeeeee/HomeAgent/internal/system"
+)
+
+// runFailbackRecovery 在 failback worker 内执行 L1 恢复梯子：
+//   probe(rescue 源) → 还原 DNS/proxy → probe → 还原 LLM 配置快照+ReloadFromConfig → probe
+// 成功后写 recovery_result.json 并以 exitRecovered 退出；失败以 exitRecoveryFailed 退出。
+// 全程确定性、无 token 消耗（除 rescue 源 QuickChat 复检外）。
+func runFailbackRecovery(dataDir string, cfgReg *internalConfig.ConfigRegistry, lua *luaVM.VM, providerMgr *agentAPI.ProviderManager, baseAPIKey string) {
+	applySystemConfig(loadGuardConfig(dataDir)) // 发行版/部署路径适配：guard.yaml system.*
+	task, err := recovery.LoadTask(recovery.TaskPath(dataDir))
+	if err != nil {
+		log.Printf("[failback] no recovery task, skipping: %v", err)
+		os.Exit(exitRecoveryFailed)
+	}
+	if task.RescueSource.Name == "" || task.RescueSource.BaseURL == "" {
+		log.Printf("[failback] rescue source missing in task, cannot recover")
+		os.Exit(exitRecoveryFailed)
+	}
+	log.Printf("[failback] recovery task: attempt %d/%d rescue=%s", task.Attempt, task.MaxAttempts, task.RescueSource.Name)
+
+	timeout := 120 * time.Second
+	if d, err := time.ParseDuration(task.AttemptTimeout); err == nil && d > 0 {
+		timeout = d
+	}
+
+	// 注册 rescue 源并设为默认（锚定 IP 直连，绕开被破坏的 DNS/代理）
+	key := task.RescueSource.APIKey
+	if key == "" {
+		key = baseAPIKey
+	}
+	rescueProvider := agentAPI.NewLuaAdaptedProvider(agentAPI.BaseConfig{
+		Model:         task.RescueSource.Model,
+		BaseURL:       task.RescueSource.BaseURL,
+		APIKey:        key,
+		Temperature:   0.7,
+		MaxTokens:     512,
+		ContextWindow: 8192,
+	}, lua, task.RescueSource.Name, task.RescueSource.Adapter)
+	providerMgr.Register("rescue", rescueProvider)
+	if err := providerMgr.SetDefault("rescue"); err != nil {
+		log.Printf("[failback] set rescue default: %v", err)
+	}
+
+	probe := func(ctx context.Context, label string) (bool, error) {
+		ctx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()
+		resp, err := providerMgr.QuickChat(ctx, "回复 OK")
+		if err != nil {
+			log.Printf("[failback] probe(%s): %v", label, err)
+			return false, nil
+		}
+		log.Printf("[failback] probe(%s): reachable, reply=%q", label, truncateStr(resp.Content, 60))
+		return true, nil
+	}
+
+	restoreNet := func() ([]string, error) {
+		base, err := system.LoadNetworkBaseline(dataDir)
+		if err != nil {
+			return nil, fmt.Errorf("load network baseline: %w", err)
+		}
+		changed, err := base.RestoreFiles()
+		if err != nil {
+			return nil, err
+		}
+		if len(changed) > 0 {
+			log.Printf("[failback] restored network files: %v", changed)
+		}
+		return changed, nil
+	}
+
+	restoreCfg := func() (int, error) {
+		snapPath := filepath.Join(dataDir, "llm_snapshot.json")
+		snap, err := internalConfig.LoadLLMSnapshot(snapPath)
+		if err != nil {
+			return 0, fmt.Errorf("load llm snapshot: %w", err)
+		}
+		if err := cfgReg.RestoreCoreLLM(snap); err != nil {
+			return 0, err
+		}
+		if err := sdk.NewLLM(providerMgr, cfgReg, lua, baseAPIKey).ReloadFromConfig(); err != nil {
+			return 0, err
+		}
+		log.Printf("[failback] restored %d llm keys and reloaded from config", len(snap))
+		return len(snap), nil
+	}
+
+	ladder := &recovery.Ladder{
+		ResultFile:     recovery.ResultPath(dataDir),
+		Attempt:        task.Attempt,
+		Probe:          probe,
+		RestoreNetwork: restoreNet,
+		RestoreConfig:  restoreCfg,
+		Log:            log.Printf,
+	}
+	res := ladder.Run(context.Background())
+	log.Printf("[failback] recovery round result: %s", res.Quote())
+
+	if res.Success {
+		os.Exit(exitRecovered)
+	}
+	os.Exit(exitRecoveryFailed)
+}
+
+func truncateStr(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n] + "..."
+}
+
+// lastDiagSummary 读取最近一次 recovery_result 生成一行自诊断摘要（worker IPC ACK 上报）。
+func lastDiagSummary(dataDir string) string {
+	res, err := recovery.LoadResult(recovery.ResultPath(dataDir))
+	if err != nil {
+		return ""
+	}
+	return res.Quote()
+}