mirror of
https://gitcode.com/JianFeeeee/HomeAgent.git
synced 2026-09-20 17:08:09 +00:00
三层回退恢复机制(L0写前留档/L1恢复梯子/L2离线回滚)+ guard 父守护
- L0: files 插件写受保护系统路径(/etc 等)前自动留档,AbstractBeforeWrite 到 data/file_baseline - L1: failback 受限 worker 执行恢复梯子 probe→还原DNS/proxy→还原LLM配置+ReloadFromConfig→probe,N轮有界 - L2: tracker changeset 持久化原文 blob,guard 离线 RollbackFromDisk 回滚 agentfs;SystemSnapshot 支撑 - guard 父守护: 心跳 IPC(PING/ACK unix socket, 文件心跳回退)、失败计数、退出码协议(42/43/44)、最后手段 - 发行版路径适配: system.protected_paths/network_paths 可注入,默认面向主流 Linux - Windows 兼容: guard.go/failback.go 加 //go:build linux, guard_windows.go 提供 no-op 桩 - 修复: guard.yaml last_resort 键冲突、changeset Content 不落盘导致离线回滚丢原文 Build 全绿, vet 干净, system/recovery/ipc/tracker 单元测试全过
This commit is contained in:
138
cmd/homed/failback.go
Normal file
138
cmd/homed/failback.go
Normal file
@ -0,0 +1,138 @@
|
||||
//go:build linux
|
||||
|
||||
// L1 failback 恢复 worker(Linux 专属):在受限 boot 下执行恢复梯子,
|
||||
// 依赖 DNS/代理网络基线、LLM 配置快照与 reload 等 Linux 语义。
|
||||
// 非 Linux 平台不编译,由 guard_windows.go 提供桩。
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/recovery"
|
||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/system"
|
||||
)
|
||||
|
||||
// runFailbackRecovery 在 failback worker 内执行 L1 恢复梯子:
|
||||
// probe(rescue 源) → 还原 DNS/proxy → probe → 还原 LLM 配置快照+ReloadFromConfig → probe
|
||||
// 成功后写 recovery_result.json 并以 exitRecovered 退出;失败以 exitRecoveryFailed 退出。
|
||||
// 全程确定性、无 token 消耗(除 rescue 源 QuickChat 复检外)。
|
||||
func runFailbackRecovery(dataDir string, cfgReg *internalConfig.ConfigRegistry, lua *luaVM.VM, providerMgr *agentAPI.ProviderManager, baseAPIKey string) {
|
||||
applySystemConfig(loadGuardConfig(dataDir)) // 发行版/部署路径适配:guard.yaml system.*
|
||||
task, err := recovery.LoadTask(recovery.TaskPath(dataDir))
|
||||
if err != nil {
|
||||
log.Printf("[failback] no recovery task, skipping: %v", err)
|
||||
os.Exit(exitRecoveryFailed)
|
||||
}
|
||||
if task.RescueSource.Name == "" || task.RescueSource.BaseURL == "" {
|
||||
log.Printf("[failback] rescue source missing in task, cannot recover")
|
||||
os.Exit(exitRecoveryFailed)
|
||||
}
|
||||
log.Printf("[failback] recovery task: attempt %d/%d rescue=%s", task.Attempt, task.MaxAttempts, task.RescueSource.Name)
|
||||
|
||||
timeout := 120 * time.Second
|
||||
if d, err := time.ParseDuration(task.AttemptTimeout); err == nil && d > 0 {
|
||||
timeout = d
|
||||
}
|
||||
|
||||
// 注册 rescue 源并设为默认(锚定 IP 直连,绕开被破坏的 DNS/代理)
|
||||
key := task.RescueSource.APIKey
|
||||
if key == "" {
|
||||
key = baseAPIKey
|
||||
}
|
||||
rescueProvider := agentAPI.NewLuaAdaptedProvider(agentAPI.BaseConfig{
|
||||
Model: task.RescueSource.Model,
|
||||
BaseURL: task.RescueSource.BaseURL,
|
||||
APIKey: key,
|
||||
Temperature: 0.7,
|
||||
MaxTokens: 512,
|
||||
ContextWindow: 8192,
|
||||
}, lua, task.RescueSource.Name, task.RescueSource.Adapter)
|
||||
providerMgr.Register("rescue", rescueProvider)
|
||||
if err := providerMgr.SetDefault("rescue"); err != nil {
|
||||
log.Printf("[failback] set rescue default: %v", err)
|
||||
}
|
||||
|
||||
probe := func(ctx context.Context, label string) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
resp, err := providerMgr.QuickChat(ctx, "回复 OK")
|
||||
if err != nil {
|
||||
log.Printf("[failback] probe(%s): %v", label, err)
|
||||
return false, nil
|
||||
}
|
||||
log.Printf("[failback] probe(%s): reachable, reply=%q", label, truncateStr(resp.Content, 60))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
restoreNet := func() ([]string, error) {
|
||||
base, err := system.LoadNetworkBaseline(dataDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("load network baseline: %w", err)
|
||||
}
|
||||
changed, err := base.RestoreFiles()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(changed) > 0 {
|
||||
log.Printf("[failback] restored network files: %v", changed)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
restoreCfg := func() (int, error) {
|
||||
snapPath := filepath.Join(dataDir, "llm_snapshot.json")
|
||||
snap, err := internalConfig.LoadLLMSnapshot(snapPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("load llm snapshot: %w", err)
|
||||
}
|
||||
if err := cfgReg.RestoreCoreLLM(snap); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := sdk.NewLLM(providerMgr, cfgReg, lua, baseAPIKey).ReloadFromConfig(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
log.Printf("[failback] restored %d llm keys and reloaded from config", len(snap))
|
||||
return len(snap), nil
|
||||
}
|
||||
|
||||
ladder := &recovery.Ladder{
|
||||
ResultFile: recovery.ResultPath(dataDir),
|
||||
Attempt: task.Attempt,
|
||||
Probe: probe,
|
||||
RestoreNetwork: restoreNet,
|
||||
RestoreConfig: restoreCfg,
|
||||
Log: log.Printf,
|
||||
}
|
||||
res := ladder.Run(context.Background())
|
||||
log.Printf("[failback] recovery round result: %s", res.Quote())
|
||||
|
||||
if res.Success {
|
||||
os.Exit(exitRecovered)
|
||||
}
|
||||
os.Exit(exitRecoveryFailed)
|
||||
}
|
||||
|
||||
func truncateStr(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
return s[:n] + "..."
|
||||
}
|
||||
|
||||
// lastDiagSummary 读取最近一次 recovery_result 生成一行自诊断摘要(worker IPC ACK 上报)。
|
||||
func lastDiagSummary(dataDir string) string {
|
||||
res, err := recovery.LoadResult(recovery.ResultPath(dataDir))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return res.Quote()
|
||||
}
|
||||
456
cmd/homed/guard.go
Normal file
456
cmd/homed/guard.go
Normal file
@ -0,0 +1,456 @@
|
||||
//go:build linux
|
||||
|
||||
// L0/L1/L2 三层回退恢复机制的核心:父守护 guard(Linux 专属)。
|
||||
//
|
||||
// 依赖 Linux 设施:syscall.Reboot / /etc 网络基线 / overlayfs 离线回滚 /
|
||||
// unix socket IPC / systemctl 重启。Windows 等平台不编译本文件,
|
||||
// 由 guard_windows.go 提供 no-op 桩,保证 cmd/homed 跨平台可构建。
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/ipc"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/recovery"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/system"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// guardConfig 父守护配置:独立于 config.db,避开被改坏的配置数据库。
|
||||
type guardConfig struct {
|
||||
MaxRestarts int `yaml:"max_restarts"` // 连续 normal 崩溃重试上限
|
||||
HeartbeatTimeout string `yaml:"heartbeat_timeout"` // 心跳多久未更新判定卡死(如 30s)
|
||||
HeartbeatInterval string `yaml:"heartbeat_interval"` // 期待 worker 心跳频率(仅日志)
|
||||
LLMSnapshot string `yaml:"llm_snapshot"` // LLM 配置基线文件,为空用 <data>/llm_snapshot.json
|
||||
FailbackEnabled bool `yaml:"failback_enabled"` // 崩溃超限后是否进入受限 failback 启动
|
||||
LastResort string `yaml:"last_resort"` // restart_app | reboot
|
||||
RestartCommand []string `yaml:"restart_command"` // last_resort=restart_app 时的命令(如 systemctl restart homeagent)
|
||||
RebootGrace string `yaml:"reboot_grace"` // last_resort=reboot 前的缓冲
|
||||
LLM llmConfig `yaml:"llm"` // rescue 源(锚定 IP,绕开被破坏的 DNS/代理)
|
||||
Recovery recoveryConfig `yaml:"recovery"` // failback 恢复参数(N 轮有界)
|
||||
LastResortCfg lastResortConfig `yaml:"last_resort_cfg"` // 最后手段细节(snapshot_before 等)
|
||||
System systemConfig `yaml:"system"` // 发行版/部署路径适配
|
||||
|
||||
heartbeatTimeout time.Duration
|
||||
heartbeatInterval time.Duration
|
||||
rebootGrace time.Duration
|
||||
}
|
||||
|
||||
type llmConfig struct {
|
||||
Sources []recovery.RescueSource `yaml:"sources"`
|
||||
}
|
||||
|
||||
type recoveryConfig struct {
|
||||
MaxAttempts int `yaml:"max_attempts"` // failback 恢复尝试轮数上限
|
||||
AttemptTimeout string `yaml:"attempt_timeout"` // 单轮恢复超时
|
||||
KnowledgeBase string `yaml:"knowledge_base"` // 恢复知识库目录
|
||||
TriggerPrompt string `yaml:"trigger_prompt"` // 恢复 agent 的系统提示词(未知/混合分支)
|
||||
Plugins []string `yaml:"plugins"` // failback 插件集(缺省用 core.agent.failback_plugins)
|
||||
attemptTimeout time.Duration
|
||||
}
|
||||
|
||||
type lastResortConfig struct {
|
||||
SnapshotBefore bool `yaml:"snapshot_before"` // 最后手段前是否回滚 agentfs 到最近快照
|
||||
}
|
||||
|
||||
// systemConfig 发行版/部署路径适配:把 L0 写前留档保护范围与网络基线文件列表
|
||||
// 从默认的 Linux 路径换成当前部署实际路径,避免硬编码失效。
|
||||
type systemConfig struct {
|
||||
ProtectedPaths []string `yaml:"protected_paths"` // L0 写前留档保护前缀(默认 /etc/)
|
||||
NetworkPaths []string `yaml:"network_paths"` // 网络基线文件(默认 resolv.conf/hosts/environment)
|
||||
}
|
||||
|
||||
// applySystemConfig 应用发行版路径适配,供 guard 与 failback worker 共用。
|
||||
func applySystemConfig(cfg *guardConfig) {
|
||||
if len(cfg.System.ProtectedPaths) > 0 {
|
||||
system.SetProtectedPaths(cfg.System.ProtectedPaths)
|
||||
}
|
||||
if len(cfg.System.NetworkPaths) > 0 {
|
||||
system.SetNetworkPaths(cfg.System.NetworkPaths)
|
||||
}
|
||||
}
|
||||
|
||||
func defaultGuardConfig(dataDir string) *guardConfig {
|
||||
return &guardConfig{
|
||||
MaxRestarts: 3,
|
||||
HeartbeatTimeout: "30s",
|
||||
HeartbeatInterval: "5s",
|
||||
LLMSnapshot: filepath.Join(dataDir, "llm_snapshot.json"),
|
||||
FailbackEnabled: true,
|
||||
LastResort: "restart_app",
|
||||
RestartCommand: []string{"systemctl", "restart", "homeagent"},
|
||||
RebootGrace: "10s",
|
||||
Recovery: recoveryConfig{
|
||||
MaxAttempts: 3,
|
||||
AttemptTimeout: "120s",
|
||||
},
|
||||
LastResortCfg: lastResortConfig{SnapshotBefore: true},
|
||||
}
|
||||
}
|
||||
|
||||
func loadGuardConfig(dataDir string) *guardConfig {
|
||||
cfg := defaultGuardConfig(dataDir)
|
||||
path := filepath.Join(dataDir, "guard.yaml")
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return cfg
|
||||
}
|
||||
if err := yaml.Unmarshal(data, cfg); err != nil {
|
||||
log.Printf("[guard] parse %s: %v, using defaults", path, err)
|
||||
return cfg
|
||||
}
|
||||
if cfg.LLMSnapshot == "" {
|
||||
cfg.LLMSnapshot = filepath.Join(dataDir, "llm_snapshot.json")
|
||||
}
|
||||
parseDur := func(s string, def time.Duration) time.Duration {
|
||||
if s == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(s)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return d
|
||||
}
|
||||
cfg.heartbeatTimeout = parseDur(cfg.HeartbeatTimeout, 30*time.Second)
|
||||
cfg.heartbeatInterval = parseDur(cfg.HeartbeatInterval, 5*time.Second)
|
||||
cfg.rebootGrace = parseDur(cfg.RebootGrace, 10*time.Second)
|
||||
cfg.Recovery.attemptTimeout = parseDur(cfg.Recovery.AttemptTimeout, 120*time.Second)
|
||||
if cfg.MaxRestarts < 1 {
|
||||
cfg.MaxRestarts = 1
|
||||
}
|
||||
if cfg.LastResort != "reboot" && cfg.LastResort != "restart_app" {
|
||||
cfg.LastResort = "restart_app"
|
||||
}
|
||||
if len(cfg.RestartCommand) == 0 {
|
||||
cfg.RestartCommand = []string{"systemctl", "restart", "homeagent"}
|
||||
}
|
||||
if cfg.Recovery.MaxAttempts < 1 {
|
||||
cfg.Recovery.MaxAttempts = 1
|
||||
}
|
||||
// 缺省 failback 插件集
|
||||
if len(cfg.Recovery.Plugins) == 0 {
|
||||
cfg.Recovery.Plugins = []string{"webui", "pluginmgr", "recoverydiag"}
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
// runGuard 父守护主循环:拉起 worker(--role=agent),心跳探活 + waitpid,
|
||||
// 崩溃后按序 重试→LLM 配置基线恢复→failback 受限启动(N 轮有界,每轮复检)→L2 最后手段。
|
||||
func runGuard(dataDir string) {
|
||||
cfg := loadGuardConfig(dataDir)
|
||||
applySystemConfig(cfg)
|
||||
hbPath := filepath.Join(dataDir, "heartbeat")
|
||||
os.MkdirAll(filepath.Join(dataDir, "log"), 0755)
|
||||
|
||||
exe, err := os.Executable()
|
||||
if err != nil {
|
||||
log.Fatalf("[guard] resolve executable: %v", err)
|
||||
}
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
log.Printf("[guard] starting, data=%s max_restarts=%d hb_timeout=%v last_resort=%s failback_rounds=%d",
|
||||
dataDir, cfg.MaxRestarts, cfg.heartbeatTimeout, cfg.LastResort, cfg.Recovery.MaxAttempts)
|
||||
|
||||
// 启动即捕获网络基线(L0:写前留档 + L1 还原依据),并给当前 worker 恢复代理环境
|
||||
captureNetworkBaseline(dataDir)
|
||||
|
||||
failures := 0 // 连续 normal 崩溃次数
|
||||
failbackRound := 0 // 已进行的 failback 轮次
|
||||
failbackDone := false // 本轮 failback 是否已进入
|
||||
|
||||
for {
|
||||
// 决定本次 boot 模式:normal 崩溃超限 → 进入 failback
|
||||
boot := "normal"
|
||||
if cfg.FailbackEnabled && !failbackDone && failures >= cfg.MaxRestarts {
|
||||
log.Printf("[guard] %d failures >= max %d, entering failback (Safe-Mode)", failures, cfg.MaxRestarts)
|
||||
// L1 前置:恢复 LLM 基线 + 还原 DNS/proxy + 写恢复任务
|
||||
restoreLLMBaseline(dataDir, cfg.LLMSnapshot)
|
||||
restoreNetworkBaseline(dataDir)
|
||||
failbackRound = 1
|
||||
failbackDone = true
|
||||
if err := writeRecoveryTask(dataDir, cfg, failbackRound); err != nil {
|
||||
log.Printf("[guard] write recovery task: %v", err)
|
||||
}
|
||||
boot = "failback"
|
||||
} else if failbackDone {
|
||||
boot = "failback"
|
||||
}
|
||||
|
||||
// 恢复 agent 环境(代理变量)在 worker 拉起前注入
|
||||
if b, err := system.LoadNetworkBaseline(dataDir); err == nil {
|
||||
b.ApplyProxyEnv()
|
||||
}
|
||||
|
||||
outcome, spawnErr := spawnWorkerOnce(exe, dataDir, boot, hbPath, cfg, sigCh)
|
||||
if spawnErr != nil {
|
||||
log.Printf("[guard] spawn worker: %v, last_resort=%s", spawnErr, cfg.LastResort)
|
||||
doLastResort(cfg)
|
||||
failures = 0
|
||||
failbackDone = false
|
||||
failbackRound = 0
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
switch outcome {
|
||||
case workerCleanExit:
|
||||
log.Printf("[guard] worker exited cleanly, guard exiting")
|
||||
return
|
||||
case workerRestartRequested:
|
||||
log.Printf("[guard] worker requested restart, respawning")
|
||||
continue
|
||||
case workerRecovered:
|
||||
// failback 成功:重置失败轮次,交回主 agent(下一轮 normal boot)
|
||||
log.Printf("[guard] failback recovery succeeded, handing back to main agent")
|
||||
failures = 0
|
||||
failbackDone = false
|
||||
failbackRound = 0
|
||||
continue
|
||||
case workerCrash:
|
||||
// fallthrough below
|
||||
}
|
||||
failures++
|
||||
|
||||
if failbackDone {
|
||||
// failback 轮次内崩溃:读结果判定是否成功
|
||||
if res, err := recovery.LoadResult(recovery.ResultPath(dataDir)); err == nil && res.Success {
|
||||
log.Printf("[guard] failback round %d result success: %s", failbackRound, res.Quote())
|
||||
failures = 0
|
||||
failbackDone = false
|
||||
failbackRound = 0
|
||||
continue
|
||||
}
|
||||
if failbackRound >= cfg.Recovery.MaxAttempts {
|
||||
log.Printf("[guard] failback exhausted after %d rounds, L2 last_resort=%s", failbackRound, cfg.LastResort)
|
||||
if cfg.LastResortCfg.SnapshotBefore {
|
||||
rollbackAgentFS(dataDir)
|
||||
}
|
||||
doLastResort(cfg)
|
||||
failures = 0
|
||||
failbackDone = false
|
||||
failbackRound = 0
|
||||
time.Sleep(2 * time.Second)
|
||||
continue
|
||||
}
|
||||
// 进入下一轮 failback
|
||||
failbackRound++
|
||||
log.Printf("[guard] failback round %d failed, next round %d (max %d)", failbackRound-1, failbackRound, cfg.Recovery.MaxAttempts)
|
||||
if err := writeRecoveryTask(dataDir, cfg, failbackRound); err != nil {
|
||||
log.Printf("[guard] write recovery task: %v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if failures >= cfg.MaxRestarts {
|
||||
log.Printf("[guard] %d normal failures, entering failback next loop", failures)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type workerOutcome int
|
||||
|
||||
const (
|
||||
workerCleanExit workerOutcome = iota
|
||||
workerCrash
|
||||
workerRestartRequested
|
||||
workerRecovered
|
||||
)
|
||||
|
||||
// spawnWorkerOnce 拉起一个 worker 并监控到其退出。
|
||||
// 返回 (outcome workerOutcome, spawnErr error);spawnErr 非 nil 表示未能拉起进程。
|
||||
func spawnWorkerOnce(exe, dataDir, boot, hbPath string, cfg *guardConfig, sigCh chan os.Signal) (workerOutcome, error) {
|
||||
cmd := exec.Command(exe, "--role=agent", "--data="+dataDir, "--boot="+boot)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdin = nil
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return workerCrash, err
|
||||
}
|
||||
pid := cmd.Process.Pid
|
||||
log.Printf("[guard] worker up pid=%d boot=%s", pid, boot)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- cmd.Wait() }()
|
||||
|
||||
hbTicker := time.NewTicker(cfg.heartbeatInterval)
|
||||
defer hbTicker.Stop()
|
||||
|
||||
ipcClient := ipc.NewClient(dataDir)
|
||||
|
||||
for {
|
||||
select {
|
||||
case sig := <-sigCh:
|
||||
log.Printf("[guard] signal %v, stopping worker", sig)
|
||||
cmd.Process.Signal(syscall.SIGTERM)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second):
|
||||
cmd.Process.Kill()
|
||||
<-done
|
||||
}
|
||||
return workerCleanExit, nil
|
||||
case err := <-done:
|
||||
if err == nil {
|
||||
log.Printf("[guard] worker pid=%d exited cleanly", pid)
|
||||
return workerCleanExit, nil
|
||||
}
|
||||
var ee *exec.ExitError
|
||||
if errors.As(err, &ee) {
|
||||
switch ee.ExitCode() {
|
||||
case exitRestartRequested:
|
||||
log.Printf("[guard] worker pid=%d requested restart (exit %d), respawning without counting failure", pid, exitRestartRequested)
|
||||
return workerRestartRequested, nil
|
||||
case exitRecovered:
|
||||
log.Printf("[guard] worker pid=%d recovered main agent (exit %d)", pid, exitRecovered)
|
||||
return workerRecovered, nil
|
||||
case exitRecoveryFailed:
|
||||
log.Printf("[guard] worker pid=%d failback attempt failed (exit %d)", pid, exitRecoveryFailed)
|
||||
return workerCrash, nil
|
||||
}
|
||||
}
|
||||
log.Printf("[guard] worker pid=%d crashed: %v", pid, err)
|
||||
return workerCrash, nil
|
||||
case <-hbTicker.C:
|
||||
// IPC PING/ACK 存活判定(带自诊断上报),失败回退文件心跳 mtime
|
||||
if st, perr := ipcClient.Ping(cfg.heartbeatTimeout); perr != nil {
|
||||
if heartbeatStale(hbPath, cfg.heartbeatTimeout) {
|
||||
log.Printf("[guard] heartbeat stale for pid=%d (ipc: %v), treating as hang, SIGKILL", pid, perr)
|
||||
cmd.Process.Kill()
|
||||
<-done
|
||||
return workerCrash, nil
|
||||
}
|
||||
} else if st != nil && st.LastDiag != "" {
|
||||
log.Printf("[guard] worker pid=%d self-diagnosis: %s", pid, st.LastDiag)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func heartbeatStale(path string, timeout time.Duration) bool {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
// 无心跳文件:worker 刚启动或异常;宽限处理
|
||||
return false
|
||||
}
|
||||
return time.Since(fi.ModTime()) > timeout
|
||||
}
|
||||
|
||||
// restoreLLMBaseline 从文件基线恢复 core.llm.* 配置(存在才恢复)。
|
||||
func restoreLLMBaseline(dataDir, snapPath string) {
|
||||
if _, err := os.Stat(snapPath); err != nil {
|
||||
log.Printf("[guard] no llm snapshot baseline at %s, skip restore", snapPath)
|
||||
return
|
||||
}
|
||||
snap, err := internalConfig.LoadLLMSnapshot(snapPath)
|
||||
if err != nil {
|
||||
log.Printf("[guard] load llm snapshot %s: %v", snapPath, err)
|
||||
return
|
||||
}
|
||||
cfgReg := internalConfig.NewConfigRegistry(filepath.Join(dataDir, "config.db"))
|
||||
defer cfgReg.Close()
|
||||
if err := cfgReg.RestoreCoreLLM(snap); err != nil {
|
||||
log.Printf("[guard] restore llm baseline: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[guard] restored %d llm keys from %s", len(snap), snapPath)
|
||||
}
|
||||
|
||||
// captureNetworkBaseline L0:启动即捕获网络基线(DNS/hosts/代理),供 failback 还原。
|
||||
func captureNetworkBaseline(dataDir string) {
|
||||
base, err := system.CaptureNetwork()
|
||||
if err != nil {
|
||||
log.Printf("[guard] capture network baseline: %v", err)
|
||||
return
|
||||
}
|
||||
if err := system.SaveNetworkBaseline(dataDir, base); err != nil {
|
||||
log.Printf("[guard] save network baseline: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[guard] network baseline captured: %s", base.Summary())
|
||||
}
|
||||
|
||||
// restoreNetworkBaseline L1:把 resolv.conf/hosts 还原到基线(DNS/proxy 小修命中即停)。
|
||||
func restoreNetworkBaseline(dataDir string) {
|
||||
base, err := system.LoadNetworkBaseline(dataDir)
|
||||
if err != nil {
|
||||
log.Printf("[guard] no network baseline, skip DNS/proxy restore: %v", err)
|
||||
return
|
||||
}
|
||||
changed, err := base.RestoreFiles()
|
||||
if err != nil {
|
||||
log.Printf("[guard] restore network baseline: %v", err)
|
||||
return
|
||||
}
|
||||
if len(changed) > 0 {
|
||||
log.Printf("[guard] restored network files: %v", changed)
|
||||
} else {
|
||||
log.Printf("[guard] network baseline already consistent")
|
||||
}
|
||||
}
|
||||
|
||||
// writeRecoveryTask 写本轮 failback 恢复任务文件。
|
||||
func writeRecoveryTask(dataDir string, cfg *guardConfig, attempt int) error {
|
||||
task := &recovery.Task{
|
||||
Attempt: attempt,
|
||||
MaxAttempts: cfg.Recovery.MaxAttempts,
|
||||
AttemptTimeout: cfg.Recovery.AttemptTimeout,
|
||||
TriggerPrompt: cfg.Recovery.TriggerPrompt,
|
||||
KnowledgeBase: cfg.Recovery.KnowledgeBase,
|
||||
PluginList: cfg.Recovery.Plugins,
|
||||
}
|
||||
for _, s := range cfg.LLM.Sources {
|
||||
if s.Name == "rescue" || s.Name != "" {
|
||||
task.RescueSource = s
|
||||
break
|
||||
}
|
||||
}
|
||||
return recovery.SaveTask(recovery.TaskPath(dataDir), task)
|
||||
}
|
||||
|
||||
// rollbackAgentFS L2:guard 在 worker 离线时回滚 agentfs 最近快照(read changesets 逆应用)。
|
||||
func rollbackAgentFS(dataDir string) {
|
||||
workDir := filepath.Join(dataDir, "agentfs")
|
||||
trk := tracker.NewOfflineTracker(dataDir, workDir)
|
||||
n, err := trk.RollbackFromDisk()
|
||||
if err != nil {
|
||||
log.Printf("[guard] agentfs rollback: %v", err)
|
||||
return
|
||||
}
|
||||
log.Printf("[guard] agentfs rolled back %d change sets", n)
|
||||
}
|
||||
|
||||
func doLastResort(cfg *guardConfig) {
|
||||
switch cfg.LastResort {
|
||||
case "reboot":
|
||||
log.Printf("[guard] last_resort=reboot, sync + reboot in %v", cfg.rebootGrace)
|
||||
time.Sleep(cfg.rebootGrace)
|
||||
syscall.Sync()
|
||||
if err := syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART); err != nil {
|
||||
log.Printf("[guard] reboot failed (likely no privilege): %v", err)
|
||||
}
|
||||
case "restart_app":
|
||||
cmd := exec.Command(cfg.RestartCommand[0], cfg.RestartCommand[1:]...)
|
||||
cmd.Stdout = os.Stdout
|
||||
cmd.Stderr = os.Stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
log.Printf("[guard] restart_app command %s: %v", strings.Join(cfg.RestartCommand, " "), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var _ = fmt.Sprintf
|
||||
32
cmd/homed/guard_windows.go
Normal file
32
cmd/homed/guard_windows.go
Normal file
@ -0,0 +1,32 @@
|
||||
//go:build !linux
|
||||
|
||||
// L0/L1/L2 三层回退恢复机制为 Linux 专属(依赖 /etc 网络基线、overlayfs、
|
||||
// syscall.Reboot、unix socket IPC、systemctl)。在 Windows 等非 Linux 平台
|
||||
// 本文件提供 no-op 桩:guard/failback 角色不执行恢复,其余主 agent 功能照常。
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
)
|
||||
|
||||
// runGuard 桩:guard 父守护模式仅支持 Linux,其余平台打印提示并退出。
|
||||
func runGuard(dataDir string) {
|
||||
log.Printf("[guard] guard mode is Linux-only (data=%s), exiting", dataDir)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// runFailbackRecovery 桩:failback 恢复为 Linux 专属,其余平台直接退出。
|
||||
func runFailbackRecovery(dataDir string, cfgReg *internalConfig.ConfigRegistry, lua *luaVM.VM, providerMgr *agentAPI.ProviderManager, baseAPIKey string) {
|
||||
log.Printf("[failback] recovery is Linux-only (data=%s), exiting", dataDir)
|
||||
os.Exit(exitRecoveryFailed)
|
||||
}
|
||||
|
||||
// lastDiagSummary 桩:无 recovery_result 时返回空自诊断。
|
||||
func lastDiagSummary(dataDir string) string {
|
||||
return ""
|
||||
}
|
||||
@ -7,58 +7,61 @@ import (
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
agentPkg "gitcode.com/JianFeeeee/HomeAgent/internal/agent"
|
||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||
agentCore "gitcode.com/JianFeeeee/HomeAgent/internal/agent/core"
|
||||
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||
agentPkg "gitcode.com/JianFeeeee/HomeAgent/internal/agent"
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||
logpkg "gitcode.com/JianFeeeee/HomeAgent/internal/log"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/ipc"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/meta"
|
||||
logpkg "gitcode.com/JianFeeeee/HomeAgent/internal/log"
|
||||
luapkg "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/pipeline"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/social"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/meta"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/nlp"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||
cli "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/cli"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/clawhubadapter"
|
||||
cli "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/cli"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/healthcheck"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/pluginmgr"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins/webui"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/recovery"
|
||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/supervisor"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
_ "gitcode.com/JianFeeeee/HomeAgent/internal/plugins"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||
)
|
||||
|
||||
func main() {
|
||||
dataDir := flag.String("data", "", "data directory (default: auto-detect next to binary)")
|
||||
httpAddr := flag.String("webui", "", "webui listen address (default: webui.listen_addr from config)")
|
||||
cliSocket := flag.String("socket", "", "cli unix socket path (default: <data>/cli.sock)")
|
||||
role := flag.String("role", "agent", "process role: guard (父守护) | agent (工作进程)")
|
||||
boot := flag.String("boot", "normal", "agent boot mode: normal | failback (受限启动,仅 failback 插件集)")
|
||||
flag.Parse()
|
||||
|
||||
// 父守护模式:只负责拉起/守护 worker,不初始化 agent 内核
|
||||
if *role == "guard" {
|
||||
runGuard(resolveDataDir(*dataDir))
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("[homed] role=agent boot=%s", *boot)
|
||||
|
||||
if *dataDir == "" {
|
||||
exe, err := os.Executable()
|
||||
if err == nil {
|
||||
*dataDir = filepath.Join(filepath.Dir(exe), "data")
|
||||
} else {
|
||||
if exe, err := exec.LookPath(os.Args[0]); err == nil {
|
||||
*dataDir = filepath.Join(filepath.Dir(exe), "data")
|
||||
} else {
|
||||
*dataDir = "./data"
|
||||
}
|
||||
}
|
||||
*dataDir = resolveDataDir(*dataDir)
|
||||
}
|
||||
|
||||
if *cliSocket == "" {
|
||||
@ -137,6 +140,8 @@ func main() {
|
||||
cfgReg := internalConfig.NewConfigRegistry(filepath.Join(*dataDir, "config.db"))
|
||||
defer cfgReg.Close()
|
||||
cfgReg.SeedDefaults(*dataDir)
|
||||
// LLM 配置写前留档(config_set 写 core.llm.* 前自动快照),guard 恢复用基线
|
||||
cfgReg.SetLLMSnapshotFile(filepath.Join(*dataDir, "llm_snapshot.json"))
|
||||
cfg := cfgReg.ToConfig()
|
||||
|
||||
// 共享词嵌入:蒸馏提取(Phase 3 TransE 验证)与 Agent 上下文复用同一实例,
|
||||
@ -235,24 +240,24 @@ func main() {
|
||||
if err := textMem.Append(te); err != nil {
|
||||
log.Printf("[homed] text memory append: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if input != "" && memDB != nil {
|
||||
distiller.Append("agent", "user", input)
|
||||
}
|
||||
if response != "" && memDB != nil {
|
||||
distiller.Append("agent", "assistant", response)
|
||||
}
|
||||
if input != "" && memDB != nil {
|
||||
distiller.Append("agent", "user", input)
|
||||
}
|
||||
if response != "" && memDB != nil {
|
||||
distiller.Append("agent", "assistant", response)
|
||||
}
|
||||
|
||||
// 工具输出接入蒸馏管线
|
||||
for _, tr := range toolResults {
|
||||
if trMap, ok := tr.(map[string]interface{}); ok {
|
||||
if text, ok := trMap["output"].(string); ok && text != "" && memDB != nil {
|
||||
distiller.Append("agent", "tool", text)
|
||||
// 工具输出接入蒸馏管线
|
||||
for _, tr := range toolResults {
|
||||
if trMap, ok := tr.(map[string]interface{}); ok {
|
||||
if text, ok := trMap["output"].(string); ok && text != "" && memDB != nil {
|
||||
distiller.Append("agent", "tool", text)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
@ -291,6 +296,12 @@ func main() {
|
||||
}
|
||||
provider := providerMgr.Default()
|
||||
|
||||
// L1 failback:受限 worker 启动即跑恢复梯子(probe→还原DNS/proxy→还原config+ReloadFromConfig→probe),
|
||||
// 结果以退出码 exitRecovered=43 / exitRecoveryFailed=44 交回 guard,不进入主 agent 循环。
|
||||
if *boot == "failback" {
|
||||
runFailbackRecovery(*dataDir, cfgReg, luaVM, providerMgr, baseAPIKey)
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// 文档记忆 + 知识库
|
||||
// ========================================================================
|
||||
@ -383,32 +394,32 @@ func main() {
|
||||
}
|
||||
|
||||
agent := agentCore.New(agentCore.AgentConfig{
|
||||
ID: "main",
|
||||
SystemPrompt: sysPrompt,
|
||||
Provider: provider,
|
||||
ProviderManager: providerMgr,
|
||||
IO: iom,
|
||||
Memory: memDB,
|
||||
Indexer: memIdx,
|
||||
Tracker: trk,
|
||||
DocStore: docStore,
|
||||
Knowledge: ks,
|
||||
SocialStore: socialStore,
|
||||
TextMemory: textMem,
|
||||
Personality: personality,
|
||||
PluginReg: pluginReg,
|
||||
PluginDir: cfg.Plugin.Dir,
|
||||
DistillInterval: cfgReg.GetDuration("core.agent.distill_interval", 30*time.Minute),
|
||||
ArchiveInterval: cfgReg.GetDuration("core.agent.archive_interval", 60*time.Minute),
|
||||
ReviewInterval: cfgReg.GetDuration("core.agent.review_interval", 120*time.Minute),
|
||||
MergeInterval: cfgReg.GetDuration("core.agent.merge_interval", 120*time.Minute),
|
||||
ID: "main",
|
||||
SystemPrompt: sysPrompt,
|
||||
Provider: provider,
|
||||
ProviderManager: providerMgr,
|
||||
IO: iom,
|
||||
Memory: memDB,
|
||||
Indexer: memIdx,
|
||||
Tracker: trk,
|
||||
DocStore: docStore,
|
||||
Knowledge: ks,
|
||||
SocialStore: socialStore,
|
||||
TextMemory: textMem,
|
||||
Personality: personality,
|
||||
PluginReg: pluginReg,
|
||||
PluginDir: cfg.Plugin.Dir,
|
||||
DistillInterval: cfgReg.GetDuration("core.agent.distill_interval", 30*time.Minute),
|
||||
ArchiveInterval: cfgReg.GetDuration("core.agent.archive_interval", 60*time.Minute),
|
||||
ReviewInterval: cfgReg.GetDuration("core.agent.review_interval", 120*time.Minute),
|
||||
MergeInterval: cfgReg.GetDuration("core.agent.merge_interval", 120*time.Minute),
|
||||
ContextSavePath: filepath.Join(cfg.Daemon.DataDir, "memory", "context.json"),
|
||||
EmbeddingModelPath: cfgReg.GetString("core.agent.embedding_model_path", ""),
|
||||
Embedder: embedder,
|
||||
Embedder: embedder,
|
||||
StageHost: stageHost,
|
||||
EventBus: evBus,
|
||||
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
||||
InputProcessing: cfg.InputProcessing,
|
||||
EventBus: evBus,
|
||||
ThinkingEnabled: cfg.LLM.ThinkingEnabled,
|
||||
InputProcessing: cfg.InputProcessing,
|
||||
})
|
||||
|
||||
// 通过 Registry 将内核依赖注入每个插件的 PluginSDK(阶段6 将替换遗留的 util.Configure)
|
||||
@ -454,6 +465,24 @@ func main() {
|
||||
// Auto-create plugins directory (without hardcoding plugin names)
|
||||
os.MkdirAll(cfg.Plugin.Dir, 0755)
|
||||
|
||||
// failback 受限启动:仅装载 failback 插件集(webfetch/files/cmd 为内核内置,
|
||||
// 此处仅控制外部插件,默认含 recoverydiag 以便直接在受限态产出恢复结论)
|
||||
if *boot == "failback" {
|
||||
list := cfgReg.GetString("core.agent.failback_plugins", "webui,pluginmgr,recoverydiag")
|
||||
// 优先使用 guard.yaml 经过 recovery 任务下发的插件集(guard 是 failback 权威)
|
||||
if task, terr := recovery.LoadTask(recovery.TaskPath(*dataDir)); terr == nil && len(task.Plugins()) > 0 {
|
||||
list = strings.Join(task.Plugins(), ",")
|
||||
}
|
||||
var names []string
|
||||
for _, s := range strings.Split(list, ",") {
|
||||
if s = strings.TrimSpace(s); s != "" {
|
||||
names = append(names, s)
|
||||
}
|
||||
}
|
||||
pluginReg.SetLoadAllowlist(names)
|
||||
log.Printf("[homed] failback boot: plugin allowlist = %v", names)
|
||||
}
|
||||
|
||||
// Load all plugins — each scans its own dir and is loaded via factory or .so
|
||||
if err := pluginReg.Load(cfg.Plugin.Dir); err != nil {
|
||||
log.Printf("[homed] warning: load plugins: %v", err)
|
||||
@ -468,9 +497,60 @@ func main() {
|
||||
agent.Start()
|
||||
defer agent.Stop()
|
||||
|
||||
// PING/ACK 心跳服务:worker 监听 unix socket,guard 发 PING、worker 回 ACK
|
||||
// (含自诊断 kernel 状态快照),替换纯文件心跳。文件心跳保留作回退。
|
||||
ipcServer := ipc.NewServer(*dataDir, func() *ipc.Status {
|
||||
st := agent.GetKernelStatus()
|
||||
llmOK := st != nil && st.LLM.Available
|
||||
tools := 0
|
||||
if st != nil {
|
||||
tools = len(st.Tools)
|
||||
}
|
||||
uptime := int64(0)
|
||||
if st != nil {
|
||||
if d, err := time.ParseDuration(st.Uptime); err == nil {
|
||||
uptime = int64(d.Seconds())
|
||||
}
|
||||
}
|
||||
return &ipc.Status{
|
||||
PID: os.Getpid(),
|
||||
Boot: *boot,
|
||||
UptimeSec: uptime,
|
||||
LLMOK: &llmOK,
|
||||
Tools: tools,
|
||||
LastDiag: lastDiagSummary(*dataDir),
|
||||
}
|
||||
})
|
||||
if err := ipcServer.Start(); err != nil {
|
||||
log.Printf("[homed] warning: ipc heartbeat server: %v", err)
|
||||
} else {
|
||||
defer ipcServer.Stop()
|
||||
}
|
||||
|
||||
sup.SetTracker(trk)
|
||||
sup.RegisterAgent("main")
|
||||
|
||||
// 真实存活源 + 重启通道:daemon 心跳语义由此修正(lastHB 只在确认存活时更新),
|
||||
// 重启动作不再空转——清理后以特殊退出码交给 guard/systemd 重建。
|
||||
restartCh := make(chan struct{}, 1)
|
||||
sup.SetHeartbeatSource(func(id types.AgentID) (time.Time, types.HealthStatus, error) {
|
||||
st := agent.GetKernelStatus()
|
||||
if st == nil {
|
||||
return time.Time{}, types.HealthDown, fmt.Errorf("no kernel status")
|
||||
}
|
||||
h := types.HealthHealthy
|
||||
if !st.LLM.Available {
|
||||
h = types.HealthDegraded
|
||||
}
|
||||
return time.Now(), h, nil
|
||||
})
|
||||
sup.SetRestartHandler(func(id types.AgentID) {
|
||||
select {
|
||||
case restartCh <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
})
|
||||
|
||||
log.Printf("[homed] main agent started, model=%s base=%s sources=%d adapters=%d",
|
||||
cfg.LLM.Model, cfg.LLM.BaseURL, len(cfg.LLM.Sources), len(luaVM.ListAdapters()))
|
||||
log.Printf("[homed] kernel ready, waiting for plugin IO...")
|
||||
@ -481,9 +561,42 @@ func main() {
|
||||
|
||||
sigCh := make(chan os.Signal, 1)
|
||||
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
|
||||
<-sigCh
|
||||
|
||||
log.Printf("[homed] shutting down...")
|
||||
// 心跳:每 5s 触碰 <data>/heartbeat,guard 据此判定工作进程是否存活/卡死
|
||||
hbPath := filepath.Join(*dataDir, "heartbeat")
|
||||
hbStop := make(chan struct{})
|
||||
go func() {
|
||||
t := time.NewTicker(5 * time.Second)
|
||||
defer t.Stop()
|
||||
writeHB := func() {
|
||||
if f, err := os.OpenFile(hbPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err == nil {
|
||||
fmt.Fprintf(f, "t=%d\n", time.Now().Unix())
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
writeHB()
|
||||
for {
|
||||
select {
|
||||
case <-t.C:
|
||||
writeHB()
|
||||
case <-hbStop:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
restartRequested := false
|
||||
select {
|
||||
case <-sigCh:
|
||||
log.Printf("[homed] shutting down...")
|
||||
case <-restartCh:
|
||||
restartRequested = true
|
||||
log.Printf("[homed] restart requested, shutting down cleanly then exiting with code %d", exitRestartRequested)
|
||||
}
|
||||
|
||||
close(hbStop)
|
||||
pluginReg.StopAll()
|
||||
if trk != nil {
|
||||
trk.Stop()
|
||||
@ -493,4 +606,8 @@ func main() {
|
||||
}
|
||||
sup.Shutdown()
|
||||
log.Printf("[homed] stopped")
|
||||
|
||||
if restartRequested {
|
||||
os.Exit(exitRestartRequested)
|
||||
}
|
||||
}
|
||||
|
||||
30
cmd/homed/worker_exit.go
Normal file
30
cmd/homed/worker_exit.go
Normal file
@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// worker 退出码协议:guard 通过 worker 退出码判定下一步动作。
|
||||
// 该协议跨平台一致(Windows 下 guard 为 no-op,退出码仍保留)。
|
||||
const (
|
||||
exitRestartRequested = 42 // worker 清理后请求重建(不计失败轮次)
|
||||
exitRecovered = 43 // failback worker 恢复成功,guard 应重置失败轮次并交回主 agent
|
||||
exitRecoveryFailed = 44 // failback worker 单轮恢复失败,guard 计入失败并进入下一轮/最后手段
|
||||
)
|
||||
|
||||
// resolveDataDir 复用原有的数据目录探测逻辑。
|
||||
func resolveDataDir(dataDir string) string {
|
||||
if dataDir != "" {
|
||||
return dataDir
|
||||
}
|
||||
exe, err := os.Executable()
|
||||
if err == nil {
|
||||
return filepath.Join(filepath.Dir(exe), "data")
|
||||
}
|
||||
if exe, err := exec.LookPath(os.Args[0]); err == nil {
|
||||
return filepath.Join(filepath.Dir(exe), "data")
|
||||
}
|
||||
return "./data"
|
||||
}
|
||||
364
demo.md
Normal file
364
demo.md
Normal file
@ -0,0 +1,364 @@
|
||||
# HomeAgent 自愈 / Failback 架构设计与讨论全程记录 (demo.md)
|
||||
|
||||
> 本文档按讨论演进顺序记录"守护 / 保活 / 文件追踪 / 崩溃自愈 / failback"整个设计过程,
|
||||
> 含代码勘查结论、现实日志记录模式分析,以及最终定稿的架构与尚未落地的接口清单。
|
||||
|
||||
---
|
||||
|
||||
## 0. 背景与目标
|
||||
|
||||
框架目标:**内核零 IO、插件承载所有 IO**(`homed 内核 ← PluginSDK → 插件`),三层记忆 + 常用块。
|
||||
|
||||
**Failback 的定位:所有错误的一层兜底(Safe-Mode 式),而非针对单一场景。**
|
||||
|
||||
- 主 agent(全量 LLM agent)是一切骚操作的执行者,可能把自己搞到无法自愈的任意状态:
|
||||
LLM 源改坏 / 系统网络(proxy、DNS、host)破坏 / 配置文件损坏 / OOM / panic / 崩溃循环……
|
||||
这些错误无法在**同一个被污染环境内**用自身操作自救。
|
||||
- 故需要一层**脱离主 agent 坏环境的、最小厚度且独立可控的恢复层**——
|
||||
类似 Windows **安全模式 / 启动修复**:只带最小驱动集合 + 干净 LLM 源(锚定 IP),
|
||||
单一职责:**让主 agent 回到可用状态;若不可行,则做最后的系统级兜底(回滚快照 / 重启)。**
|
||||
- 它不替代主 agent 的功能,只在主 agent 无法自愈时作为最后一道防线出现。能省则省、能判定就不推理、有底即主张。
|
||||
|
||||
---
|
||||
|
||||
## 1. 现状盘点(代码勘查结论)
|
||||
|
||||
### 1.1 守护进程(`internal/supervisor/daemon.go`)
|
||||
|
||||
- 主 agent in-process 常驻,`agent.Start()` 在 `cmd/homed/main.go:468` 直接启动,**无独立进程边界**。
|
||||
- `healthLoop` → `checkAgent` 判 LLM 是否可达,判定**仅依赖 `network.Monitor` 的 `AggregateResult().LLMAPIReachable`**(`daemon.go:132`)。
|
||||
- `failCount >= MaxRetries`(默认 3)→ `handleFailure`:
|
||||
- 有 tracker → `trk.Rollback()`;失败才降级 `restartAgent`。
|
||||
- `restartAgent`(`daemon.go:166`)**只改内存状态再重新 Register,不真重启任何进程**,几乎空转。
|
||||
|
||||
关键问题:
|
||||
- 探测是系统级(HTTP/DNS/TCP),回滚只作用于 `<data>/agentfs` overlay 的 upper,**二者对象错位**。
|
||||
- `AgregateResult` 在无 LLM endpoint 时恒 healthy,机制形同虚设。
|
||||
- `lastHB` 每轮都置 `time.Now()`,`Uptime` 无意义。
|
||||
- `RollbackPolicy` 的 `HealthThreshold/CooldownPeriod/AutoRollback` 都是死字段,只用 `MaxRetries`。
|
||||
|
||||
### 1.2 文件追踪(`internal/tracker`)
|
||||
|
||||
- overlayfs 三层:`lower/upper/work → merged`(`tracker.go:156`)。
|
||||
- `captureFSState(upperDir)` 递归遍历 upper 并 sha256(`changeset.go:56`);`PreAction/PostAction` 前后 diff(`toolcall.go:86-94`)。
|
||||
- **lower 恒空**(`Init` 只 `MkdirAll`,从不填充)→ 无 canonical 基线可回滚。
|
||||
- `Rollback()` = `RemoveAll(upper)` 清空全部 changesets;`FileChange.Content`(本应存回滚原文)**从未回填**。
|
||||
|
||||
结论:overlay/tracker 对"LLM 可达性"这主场景**错位**,只能作数据兜底。
|
||||
|
||||
### 1.3 通信插件真相(`third_party/homeagent-sdk/example/qq/plugin.go`)
|
||||
|
||||
- agent 对外通信全部由**插件设置**驱动,存于 **ConfigRegistry / SQLite config.db**:
|
||||
`qq.napcat_url`、`qq.listen`、`qq.files_dir`、`qq.remote_dir`、`dm/group_policy`(`plugin.go:120-130`)。
|
||||
- 插件在 `Start()` 里 `getSetting(...)` 读设置(`plugin.go:134-143`)→ **改动配置需重载插件才生效**。
|
||||
- `cfgmgr` 提供 `config_set / config_batch_set` 可运行时改任意 core/插件配置(`cfgmgr/plugin.go:54,103`)。
|
||||
|
||||
### 1.4 LLM 源与"恢复即生效"
|
||||
|
||||
`internal/sdk/llm_impl.go:103 ReloadFromConfig()` **已存在**:
|
||||
- `cfg := cfgReg.ToConfig()` 从 config.db 重建(含 `core.llm.sources.*`,见 `registry.go:590`)
|
||||
- `mgr.Reset()` → 逐源 `NewLuaAdaptedProvider` → 重设默认。
|
||||
|
||||
即:**LLM 源的"恢复即生效"钩子已经具备**,缺的是"快照 + 探测 + 触发"三件事。
|
||||
|
||||
---
|
||||
|
||||
## 2. 现实环境:日志记录模式内参
|
||||
|
||||
> 看真实 systemd 托管的 HomeAgent(`/home/newqqagent`)日志,**目的是弄清现有的日志模型**
|
||||
> (写哪、什么格式、工具调用打在哪),为 failback / recoveryDiag 的 `diag_log_scan` 提供准确的解析依据。
|
||||
|
||||
### 2.1 systemd 托管现状(样例)
|
||||
|
||||
```
|
||||
homeagent.service: Type=simple, ExecStart=/usr/local/bin/homed -data /home/newqqagent, Restart=always, RestartSec=10
|
||||
llm-mock.service: ExecStart=/usr/bin/python3 /opt/llm-mock/mock_server.py, Restart=always, RestartSec=3
|
||||
```
|
||||
|
||||
- 实测数据区:`/home/newqqagent/` 下有 `log/`、`config.db`、`agentfs/`(overlay merged)、`snapshots/`、`changesets/`、`knowledge/`、`memory/`、`plugins/`、
|
||||
`cli.sock`、`adapters/`、`homed.log`、`memos.json` 等——**日志以独立子目录 `log/` 存放,与配置/快照/knowledge 分置**。
|
||||
- 启动段确认:`[files] started, sandbox: /`(**files 沙箱=全主机 `/` 实锤**);`main agent started, model=mock-model base=http://127.0.0.1:18080/v1 sources=3 adapters=8`(LLM 走本地 mock)。
|
||||
|
||||
### 2.2 日志目录与格式(核心)
|
||||
|
||||
- **目录配置**:`core.log.path`,默认 `<dataDir>/log`(`internal/config/registry.go:415,508`)。
|
||||
- **单次运行文件**:每次启动新建 `homed_<YYYY-MM-DD_HH-MM-SS>.log`(`cmd/homed/main.go:75`),
|
||||
写入 `logDir` 下;`log.SetOutput(io.MultiWriter(os.Stderr, logFile))`(`main.go:80`)——
|
||||
**同时进 stderr(systemd 捕获到 journald/`journalctl -u`)与文件**。
|
||||
- **格式**:标准 Go `log.Printf`,即 `YYYY/MM/DD HH:MM:SS file.go:line: [module] message`。
|
||||
用户可看文件,也可用 `journalctl -u homeagent.service` 看同一来源(同一行)。
|
||||
- **层级压缩 + 保留**(`internal/log/manager.go:26-28` + `compressor.go`):
|
||||
- 周度压缩 → `week_<year>-W<ww>.tar.gz`;月度 → `month_<yyyy-mm>.tar.gz`;年度 `year_*.tar.gz`;
|
||||
raw 文件正则 `^homed_(\d{4}-\d{2}-\d{2})_\d{2}-\d{2}-\d{2}\.log$`(`compressor.go:16`)。
|
||||
- 保留策略:`core.log.retention`(default forever)、`core.log.retention_months`(default 3),
|
||||
`applyRetention` 只留当前周 + 近 N 月(`retention.go`)。
|
||||
实测:`log/` 下即为 `homed_2026-08-03_08-03-38.log` + `month_2026-*.tar.gz` + `week_2026-W31.tar.gz`,与代码一致。
|
||||
|
||||
### 2.3 工具调用日志打在哪儿(进程主循环 `internal/agent/core/process.go`)
|
||||
|
||||
| 位置 | 日志行内容 | 备注 |
|
||||
|---|---|---|
|
||||
| `process.go:36` | `[agent] tool call loop start, max_ctx=… target=… fixed=… mem=… ctx=… N tools, M events, personality=X, docs=K` | 每轮循环开头上下文统计 |
|
||||
| `process.go:196` | `[agent] executing tool: <name> (plugin=<p>, id=<id>)` | **只记工具名/插件/id,不记 args** |
|
||||
| `process.go:226` | `[agent] tool <name> result: <截断100字符>` | 结果截断到 100 字符(`truncateStr`)|
|
||||
| `process.go:218` | `[agent] skip tool <name>: plugin <name> unhealthy` | 插件崩溃态跳过 |
|
||||
| `process.go:89/109/121/125` | LLM fallback:`trying provider %q (#%d)` / `switched active provider` / `provider %q marked unavailable (HTTP %d)` / `provider %q failed` | 主循环内 LLM 商可观测 |
|
||||
| `toolcall.go:20` | `[agent] tool %s panic: %v` + `debug.Stack()` | 工具 panic + 完整栈 |
|
||||
| `toolcall.go:41` | `[agent] tool %s timed out after 60s` | 60s 超时 |
|
||||
| `toolcall.go:92` | `[agent] tool %s changed %d files (changeset: %s)` | overlay changeset 摘要 |
|
||||
| 插件侧 | Lua 插件 `sdk.log` → `print("[lua-plugin] <level>: <msg>")` | 模板见 `cmd_debug.go:74`/`templates.go` |
|
||||
|
||||
- 完整的工具**入参/结果**在 EventBus 事件 `EventToolCall`(`{tool, plugin, args, result, status}`,`process.go:184/205`)而非文件日志——**文件日志只是执行/结果的摘要指针**(结果被截断)。
|
||||
- 重要观察:日志里未见 shell/cmd 之类的操作系统执行类调用摘要落盘(`[cmd]` 只在工具结果里),
|
||||
需要的话由 `diag_log_scan` 对 `executing tool: cmd_*` 前缀做签名匹配即可。
|
||||
|
||||
### 2.4 崩溃 / 重启观察
|
||||
|
||||
- `NRestarts=0`;MainPID 自 08-03 起稳定 3330844。曾出现**真实重复 panic**(pid 3310036):
|
||||
`[stage] handler panic: runtime error: invalid memory address or nil pointer dereference`(03:23 / 05:23 / 07:23,约每 2h),
|
||||
被 `stages.go:139` 的 `RunStage` recover 吞掉 → **进程未真崩**,systemd 未见重启。
|
||||
- 08:03:38 有过一次干净 `[homed] stopped` → systemd `Started` → pid 3310036 → 3330844。
|
||||
- 对 failback 的意义:现有崩溃防护全赖 **in-process recover**,真实进程级崩溃从未被监督;
|
||||
且当前 `Restart=always` 由 systemd **直绑 worker 且无 StartLimit**——一旦真崩并陷入循环,
|
||||
systemd 每 10s 反复拉起,没有独立 failback/取证层。→ guard 取代点在此。
|
||||
|
||||
---
|
||||
|
||||
## 3. 设计演进(讨论全过程)
|
||||
|
||||
### 3.0 起点:`internal/supervisor` + `internal/tracker` 我是"保活 + 文件追踪"
|
||||
|
||||
- 保活 = 网络健康感知 + 内存态重置;追踪 = overlayfs 变更集 + `Rollback` 清空。
|
||||
- 意图:LLM 不可达 → 回滚 agent 文件改动 → 自愈。
|
||||
|
||||
### 3.1 第一次纠正:回滚对象错位
|
||||
|
||||
- 回滚只作用于 overlayfs upper;而真正能改坏网络的路径(files 默认 `/`,可写 `/etc/resolv.conf`、`/etc/hosts`、代理配置)与 LLM 配置(config.db、adapters 目录)**都不在 overlay 内**。
|
||||
- → 检测命中但回滚删错对象,闭环在"回滚"这一环断掉。
|
||||
|
||||
### 3.2 第二次纠正:授权不能靠外部插件主动配合
|
||||
|
||||
- 阶段管道是被动通知(`RunStage` 收集 error,不主动拒绝),拒绝依赖各插件 handler。
|
||||
- 插件是外部不可控对象 → "安全 = 插件主动授权放行"不成立。
|
||||
- → 安全应做**默认拒绝**,由内核在 `executeToolCall` 分发点按 ToolDef 的 capability 裁决;配置快照 + replug 只兜底"可文件化"改动。
|
||||
- 中途又修正:真正的祸首不是 IO 组件,而是 LLM 源/系统网络配置;IO 组件崩溃是正交偶发轴,不并进来。
|
||||
|
||||
### 3.3 第三次纠正:LLM 源才是核心(config.db + ReloadFromConfig 已具钩子)
|
||||
|
||||
- `llm_set_source` 只切内存默认;真正改坏 LLM 源靠 `cfgmgr.config_set` 写 config.db `core.llm.sources.*`。
|
||||
- `ReloadFromConfig()` 已能"恢复即生效" → 只需补:**LLM 配置快照 + 真实 liveness 探测(QuickChat)+ 在 handleFailure 里触发 RFO 复检**。
|
||||
- 但用户进一步点明:**重点是 agent 改了系统网络配置文件(proxy/DNS)** —— 这类连救援 LLM 都连不上。
|
||||
|
||||
### 3.4 定稿架构一:lastFailback(独立进程,第一道防线)
|
||||
|
||||
- 独立进程 + agent 碰不到的 root:0600 配置 + 最小插件集(文件读写 + cmd)+ 单一任务。
|
||||
- 用**锚定 IP/干净 DNS** 的 LLM 源绕开坏掉的 DNS/proxy。
|
||||
- 若 failback 也失败 → agentfs 回滚最近快照 + 重启计算机(应用快照前系统配置)。
|
||||
- 兼顾 systemd 托管冲突(主 agent 崩溃循环被 systemd 反复拉起 → failback 无限增殖)。
|
||||
|
||||
### 3.5 定稿架构二:去掉强 systemd 依赖 → 独立 guard 守护进程
|
||||
|
||||
- systemd 只绑定**静态、极小、永驻的 guard**;guard 作为父进程拉起 worker(`homed` 全量 agent)。
|
||||
- guard 通过 IPC(unix socket / 匿名 pipe)收 worker 心跳;
|
||||
父进程 `waitpid` 天然 reap 崩溃/OOM/信号退出。
|
||||
- worker 可崩可换,systemd 完全看不见 → **failback 唯一性 + 无风暴天然成立**。
|
||||
- guard 抗 OOM:`oom_score_adj` 拉低,优先杀 worker 而非 guard。
|
||||
|
||||
### 3.6 定稿架构三:更纯粹 —— 同一二进制的两个 role,非独立二进制
|
||||
|
||||
- **不新建二进制**。`homed` 拆两个入口:
|
||||
- `homed --role=guard`:父守护进程,先起,负责拉起/监测/探活/裁决/恢复。
|
||||
- `homed --role=agent`:主 agent(工作进程,guard 的子进程)。
|
||||
- `homed --role=agent --boot=failback`:恢复用 agent(受限 bootstrap)。
|
||||
- guard 复用现有 homeagent 基础设置;检测到崩溃时拉起 failback agent,只加载
|
||||
**webfetch + 文件读写 + cmd** 三个插件,外加 **恢复知识库插件** 与 **常见错误检测插件**,
|
||||
复用 agent 核心以 `trigger_prompt` 初始化,要求其"尝试恢复主 agent"。
|
||||
- guard 配置**独立 YAML**,不复用 config.db(逃生舱知识必须脱离 agent 可达区)。
|
||||
|
||||
### 3.7 recoveryDiag:崩溃取证 / 根因定位插件(省 token 关键)
|
||||
|
||||
- 铁律:**工具返回结论,不返回原文**(签名式/统计式/确定性排序)。
|
||||
- 工具集:
|
||||
| 工具 | 作用 |
|
||||
|---|---|
|
||||
| `diag_triage` | exit code/信号+uptime+头部嫌疑,快速粗分"进程死亡 vs 配置类不可达" |
|
||||
| `diag_db` | config.db integrity_check + LLM 源解析校验,逐项 ok/fail |
|
||||
| `diag_log_scan` | 时间窗内命中已知错误签名(panic/provider failed/unreachable/sql/OOM)|
|
||||
| `diag_delta` | 崩溃前 config/agentfs 与 last-good 快照 diff("改了什么")|
|
||||
| `diag_loc` | 综合正交,输出按因果强度排序的定位结论 + 推荐动作 |
|
||||
|
||||
- 崩溃类别 → 恢复分支决策表:
|
||||
| 结论类 | 走分支 |
|
||||
|---|---|
|
||||
| 配置损坏类 | 还原 config 快照 + ReloadFromConfig + 拉活主 agent(无需 agent 推理)|
|
||||
| 系统网络类 | 还原 DNS/proxy → 重载主 agent(第一步小修命中即停)|
|
||||
| 进程失稳类(OOM/panic)| 不还原配置,检查内存/泄漏 → 重建 worker |
|
||||
| 未知/混合 | 放开 webfetch/知识库,用 rescue 源 + diag_loc 摘要最小推理 |
|
||||
|
||||
- 只有"未知/混合"消耗 token,前几类近乎 0 token。
|
||||
- 结论落盘 `recovery_kb/diag_<ts>.json`,回流知识库,同类崩溃下次直接命中,越用越省。
|
||||
|
||||
### 3.8 三层防御总览(最终)
|
||||
|
||||
```
|
||||
L0 平时:核心只读探活 + 写前快照(config_set 写 core.llm.* 前、files 写 /etc 前自动留档)
|
||||
L1 failback(guard 拉起,Safe-Mode 式兜底):按诊断分支逐类恢复——
|
||||
还原 DNS/proxy → 还原 config 快照 + ReloadFromConfig → QuickChat 复检 → 拉起主 agent
|
||||
L2 最后手段:agentfs 回滚最近快照 + 重启(应用快照前系统配置)
|
||||
```
|
||||
|
||||
- failback 是**所有错误(LLM 不可达 / 网络 / 配置损坏 / OOM / panic / 崩溃循环)的统一兜底层**,
|
||||
并非只针对某一条;`diag_*` 决定它走哪条恢复路径。
|
||||
|
||||
---
|
||||
|
||||
## 4. 最终架构(定稿)
|
||||
|
||||
### 4.1 进程拓扑(同一二进制,两个 role)
|
||||
|
||||
```
|
||||
systemd ──▶ homed --role=guard # 父守护进程,永驻、静态、极小
|
||||
├─ exec ──▶ homed --role=agent # 主 agent(可崩)
|
||||
└─ exec ──▶ homed --role=agent --boot=failback # 恢复用 agent
|
||||
```
|
||||
|
||||
- guard:先起,持有恢复知识(锚定源 / DNS/proxy 还原 / 配置快照 / failback 逻辑)。
|
||||
- worker:guard 子进程,心跳经 IPC,崩溃由 guard reap + 判型。
|
||||
- failback agent = 受限启动(webfetch+files+cmd + 恢复知识库 + recoveryDiag),单一任务"恢复主 agent",N 轮有界。
|
||||
|
||||
### 4.2 guard 独立 YAML 示例
|
||||
|
||||
> 现状实现(§5 已完成):`guard.yaml` 已落地为 `max_restarts / heartbeat_timeout / heartbeat_interval / llm_snapshot / failback_enabled / last_resort / restart_command / reboot_grace` 子集(`cmd/homed/guard.go`),恢复梯子=重试→LLM 基线恢复→failback 受限启动→last_resort。下表的 rescue 源 / trigger_prompt / N 轮 failback 推理是目标态,未实现。
|
||||
|
||||
```yaml
|
||||
role: guard
|
||||
|
||||
llm:
|
||||
sources:
|
||||
- name: rescue
|
||||
base_url: http://1.2.3.4:8080 # 锚定 IP 直连,绕开被破坏的 DNS/代理
|
||||
api_key: ${GUARD_RESCUE_KEY}
|
||||
adapter: ... # 锚定/SNI 型适配器
|
||||
|
||||
recovery:
|
||||
max_attempts: 4 # 可配置尝试轮次
|
||||
attempt_timeout: 120s
|
||||
knowledge_base: /opt/homeagent/recovery/
|
||||
trigger_prompt: "你是恢复 agent,唯一任务:让主 agent 恢复运行。优先还原 DNS/代理,再重载 LLM 源…"
|
||||
plugins: [webfetch, files, cmd]
|
||||
|
||||
last_resort:
|
||||
action: reboot # restart_app | reboot
|
||||
snapshot_before: true
|
||||
```
|
||||
|
||||
### 4.3 guard 恢复状态机(N 轮有界)
|
||||
|
||||
```
|
||||
guard 检测( exit≠0 | OOM | 心跳超时 | guard 锚定源探活失败 )
|
||||
1. 固化追溯:exit/信号、panic、journal、OOM 上下文 → 永久区
|
||||
2. 拉起 failback agent(受限插件 + rescue 源 + trigger_prompt + 知识库 + recoveryDiag)
|
||||
for attempt in 1..N:
|
||||
(可选先 diag_triage/diag_loc 判型)
|
||||
failback 尝试恢复
|
||||
guard 每轮复检主 agent 是否可达/存活
|
||||
├─ 成功 → 结束,交回主 agent
|
||||
└─ 超时/失败 → kill 重建,进入下一轮
|
||||
3. N 轮未成 → 取消 failback agent
|
||||
→ agentfs 回滚崩溃前最近快照
|
||||
→ 依 yaml 执行最后手段:restart_app 或 reboot
|
||||
```
|
||||
|
||||
### 4.4 systemd 绑定(极简,杜绝风暴)
|
||||
|
||||
```
|
||||
[Unit] # guard
|
||||
OnFailure=... # 备用,通常不触发(guard 稳定)
|
||||
|
||||
[Service] # guard
|
||||
Restart=always # guard 静态稳定 → 几乎不重启
|
||||
ExecStart=/usr/local/bin/homed --role=guard ...
|
||||
# No StartLimit needed for loop 情况;guard 不崩
|
||||
```
|
||||
|
||||
- 主 agent 崩 → 只触发 guard 内部 failback;systemd 仅看 guard,看不到 worker 崩溃循环。
|
||||
- failback 唯一性 + 无启动风暴:由"guard 永驻、唯一裁决"天然保证。
|
||||
- guard 抗 OOM:`oom_score_adj` 拉低。
|
||||
|
||||
---
|
||||
|
||||
## 5. 尚未落地的接口 / 下一步
|
||||
|
||||
**已完成**:
|
||||
|
||||
`recoverydiag` 快速检查插件(`third_party/homeagent-sdk/example/recoverydiag/`,外部插件)。
|
||||
- 五件套全实现:`diag_triage`(退出码/信号/存活粗分)、`diag_db`(config.db integrity_check + LLM 源字段校验,sqlite3 CLI 优先、缺失回退内核 Settings)、`diag_log_scan`(日志签名按类计数)、`diag_delta`(baseline vs 现状 diff)、`diag_loc`(四项结论正交排序 + 推荐恢复动作)。
|
||||
- 全部确定性、返回结论非原文、`NoMemory`;工具实际名带插件前缀 `recoverydiag_diag_*`。
|
||||
- 已通过 go vet + 6 个单测(对真实 config.db/日志跑通:3 个 LLM 源全 ok、日志命中 228 行主导 provider/fatal),并用**仓库内重建的 plugindev** 打出 `dist/recovery_diagnostics_linux_amd64.hmap`,装进运行实例(`/home/newqqagent/plugins/recoverydiag/`)加载成功、注册 5 工具。
|
||||
- **结论落盘 + 知识库回流**:`diag_loc` 增 `persist`(缺省 true)→ 写 `<data_dir>/recovery_kb/diag_<ts>.json`(可配 `recovery_kb_dir`),并经 `sdk.Knowledge().Add` 以 `diag:<cause>:<ts>` 回流知识库(同类崩溃下次直接命中,越用越省);失败不阻塞工具。新增 `TestDiagLocPersist`。
|
||||
- 顺带修复:仓库内 `plugindev` 需重编译(`/usr/local/bin/plugindev` 是旧版、桥模板缺 `InjectInputSync`);重编译见 `third_party/homeagent-sdk/tools/plugindev`,`go build -o ... .`。
|
||||
- 注意:本环境 `snapshots/`、`changesets/` 均为空(direct 模式无基线)→ `diag_delta` 需显式传入 baseline_dir;未来接 guard 时由快照解包目录提供。
|
||||
|
||||
`ConfigRegistry` 快照钩子(`internal/config/registry.go`)。
|
||||
- `SnapshotCoreLLM()`:抓全部 `core.llm.*` 键值快照;`RestoreCoreLLM(snap)`:精确还原(快照内键回写、快照外当前键删除)。
|
||||
- `SetLLMSnapshotFile(path)`:写前自动留档——此后任意写 `core.llm.*` 键先把当前 LLM 配置整体快照到该文件(guard 恢复的外部基线);homed 启动即挂 `<data>/llm_snapshot.json`。
|
||||
- 文件持久化对:`SaveLLMSnapshot/LoadLLMSnapshot`。新增 `TestSnapshotRestoreCoreLLM`、`TestLLMSnapshotFile`、`TestSetLLMSnapshotFile`。
|
||||
|
||||
`homed --role{guard,agent}` 入口拆分 + `--boot=failback` 受限插件集(`cmd/homed/`)。
|
||||
- `--role=guard` 父守护(永驻):读独立 `<data>/guard.yaml`(避开被改坏的 config.db),拉起 worker(`--role=agent`)、心跳探活 + waitpid 收割、信号转发停机。
|
||||
- `--role=agent` 工作进程:默认启动全插件;`--boot=failback` 走插件白名单(`core.agent.failback_plugins`,缺省 `webui,pluginmgr,recoverydiag`),内核 webfetch/files/cmd 仍内置可用。
|
||||
- guard 恢复梯子(已端到端实测):连续 `max_restarts` 次 normal 崩溃 → `restoreLLMBaseline`(从 llm_snapshot.json 恢复 core.llm.*)→ failback 受限启动 → failback 也崩 → `last_resort`(restart_app / reboot)。
|
||||
- 心跳:worker 每 5s 触碰 `<data>/heartbeat`(agent 角色 goroutine),guard 以 mtime 判定卡死(超 `heartbeat_timeout` 即 SIGKILL 计入崩溃)。
|
||||
- 插件注册表加 `SetLoadAllowlist(names)`:白名单外插件(含已注册工厂)一律跳过,failback 40 工具 → 4 工具实测通过。
|
||||
|
||||
现状 bug 修复(supervisor/network/tracker)。
|
||||
|
||||
- **monitor 无 endpoint 恒 healthy**(`internal/network/monitor.go` + `pkg/types`):`NetworkCheckResult` 增 `EndpointsConfigured`;无探活端点时不再谎报 `LLMAPIReachable=true`(置 false + Error),`NewMonitor` 初始化空切片消除启动竞态;daemon 仅在配置了端点时才据此判定降级。探活端点新增 `core.defaults.llm_endpoints`(逗号分隔,留空自动取 LLM 源 base_url),生产从此健康检查有真实目标。
|
||||
- **lastHB 恒置 now**(`internal/supervisor/daemon.go`):`checkAgent` 接入真实存活源 `SetHeartbeatSource`(homed 注册为 agent core `GetKernelStatus`),只在确认 agent 存活时更新 `lastHB`;无源置 `HealthUnknown`,存活源丢失置 `HealthDown` 且不再刷新 lastHB。
|
||||
- **restartAgent 只改内存空转**:增 `SetRestartHandler`(homed 注册为"清理后以 `exitRestartRequested=42` 退出"),不再假装成功;guard 把 42 识别为"请求重建"(`workerRestartRequested`,不计失败轮次直接重建),无 guard 时 systemd `Restart=always` 兜底。无 handler 时仅内存复位并打日志。
|
||||
- **tracker 三缺陷**(`internal/tracker/`):`captureFSStateWithContent` 为 before 基线捕获原文(上限 8MB)→ `diffStates` 对 modified/deleted 回填 `FileChange.Content`(回滚用原文);新增 `RollbackLatest()` 定向撤销最近一条 changeset;`Rollback()` 改为按时间逆序逐条逆应用(还原被改/被删文件原文、删除新增),无 changeset 时才退回整目录重置。新增 6 个测试覆盖。
|
||||
|
||||
剩余:
|
||||
|
||||
- guard ↔ agent 心跳 IPC 升级为带自诊断上报的 `PING/ACK`(当前为文件心跳 + 退出码)。
|
||||
- supervisor 适配成 guard 的探测/裁决逻辑;`ReloadFromConfig()` 复用为"恢复即生效"。
|
||||
- 生产实例迁移:编译新 homed、改 systemd 只托管 guard(`--role=guard`),确认 failback 插件(recoverydiag)就位。
|
||||
|
||||
---
|
||||
|
||||
## 附录:真实日志节选(systemd 托管示例,`/home/newqqagent`)
|
||||
|
||||
```
|
||||
# systemd unit
|
||||
homeagent.service: Type=simple, ExecStart=/usr/local/bin/homed -data /home/newqqagent, Restart=always, RestartSec=10
|
||||
llm-mock.service: ExecStart=/usr/bin/python3 /opt/llm-mock/mock_server.py, Restart=always, RestartSec=3
|
||||
|
||||
# 日志文件与格式(log.Printf 标准格式)
|
||||
2026/08/03 08:03:38 main.go:80: [homed] logging to /home/newqqagent/log/homed_2026-08-03_08-03-38.log
|
||||
2026/08/03 08:03:38 daemon.go:61: [homed] daemon started successfully
|
||||
2026/08/03 08:03:39 tracker.go:65: [tracker] initialized (work=/home/newqqagent/agentfs)
|
||||
|
||||
# 工具调用摘要(process.go)
|
||||
2026/08/03 10:03:52 process.go:36: [agent] tool call loop start, max_ctx=32768 target=26214 fixed=1306 mem=8302 ctx=16606 176 tools, 31 events, personality=true, docs=5589
|
||||
... process.go:196: [agent] executing tool: <name> (plugin=<p>, id=<id>)
|
||||
... process.go:226: [agent] tool <name> result: <截断100字符>
|
||||
|
||||
# 启动 & 沙箱
|
||||
homed[3330844]: [files] started, sandbox: /
|
||||
homed[3330844]: main agent started, model=mock-model base=http://127.0.0.1:18080/v1 sources=3 adapters=8
|
||||
|
||||
# 重复 in-process panic(被 RunStage recover 吞掉,未进程级崩溃)
|
||||
homed[3310036]: [stage] handler panic: runtime error: invalid memory address or nil pointer dereference # 03:23 / 05:23 / 07:23
|
||||
|
||||
# 一次性干净重启(systemd 手动/触发 Started,pid 3310036 → 3330844)
|
||||
homed[3310036]: [homed] stopped
|
||||
systemd[1]: Stopped homeagent.service - HomeAgent - 24/7 AI Butler.
|
||||
systemd[1]: Started homeagent.service - HomeAgent - 24/7 AI Butler.
|
||||
|
||||
# 运行状态
|
||||
systemctl show homeagent.service -p NRestarts → 0
|
||||
systemctl show homeagent.service -p MainPID → 3330844(自 08-03 起稳定)
|
||||
|
||||
# 归档
|
||||
/home/newqqagent/log/: homed_2026-08-03_08-03-38.log + week_2026-W31.tar.gz + month_2026-*.tar.gz
|
||||
```
|
||||
284
docs/zh/plan.md
Normal file
284
docs/zh/plan.md
Normal file
@ -0,0 +1,284 @@
|
||||
# WebUI 布局与配置归位修复计划
|
||||
|
||||
## 一、背景
|
||||
|
||||
上一轮 SDK 接口化改造完成并部署后,用户指出三个问题:
|
||||
|
||||
1. **WebUI 窄屏布局损坏**:顶部 `<nav>` 为桌面式横排(标题 + 6 tab + 连接指示器 + 语言 + 主题),
|
||||
窄屏断点仅缩小字号不换行,`body { overflow-x:hidden }` 直接把溢出的 tab 裁掉不可点击。
|
||||
2. **OpenClaw skills 目录被注册为核心配置**:`core.skills.path`("OpenClaw 技能存储目录")注册在核心
|
||||
配置表(`internal/config/registry.go`),但全仓无任何读取方(死配置);实际生效路径是
|
||||
clawhubadapter 自己的 `skills_dir` 配置(`config_clawhubadapter` 表 + `core.daemon.data_dir`/skills 兜底)。
|
||||
技能目录是 clawhubadapter 适配加载的领域,不应属于核心配置。
|
||||
3. **clawhubadapter 加载的微信插件成为独立配置项**:设置页出现 `channels.wechat.*`(核心表)、
|
||||
`plugin.wechat.*`(config_wechat 表)、`config_openclaw_weixin`(空表)等多处微信配置,
|
||||
全部为历史残留——当前代码零引用,OC 技能的配置实际在其自身 `~/.openclaw/openclaw.json`。
|
||||
设置页会把核心表全部键 + 全部插件表当作配置组展示,导致残留以"独立配置项"形态出现。
|
||||
|
||||
## 二、修复计划
|
||||
|
||||
| # | 动作 | 位置 | 风险 |
|
||||
|---|------|------|------|
|
||||
| A | 窄屏导航修复:<768px 下 nav 横向滚动、h1 缩写、连接指示器简化;body 溢出裁切改为 nav 内滚动 | `cmd/gui/renderer/style.css` | 无 |
|
||||
| B | 删除 `core.skills.path` 核心配置注册(set + RegisterDef 两处) | `internal/config/registry.go` | 无(无读取方) |
|
||||
| C | 备份后清理残留配置:`channels.wechat.*` 键、`config_wechat` / `config_openclaw_weixin` / `config_openclaw` 表(含微信 token,先备份) | 生产库 `/home/newqqagent/config.db` | 低(当前代码不读) |
|
||||
|
||||
## 三、实施记录
|
||||
|
||||
### 步骤 A:webui 窄屏导航修复(已完成)
|
||||
- 修复对象为 webui HTTP 服务真正前端 `internal/plugins/webui/dashboard.html`(`go:embed` 内嵌,
|
||||
登录后 `/` 返回,104KB;cmd/gui 是独立 electron 客户端,非 webui 一部分)。
|
||||
- `<768px` 断点:`nav { overflow-x:auto; scrollbar-width:none; flex-wrap:nowrap }` + `::-webkit-scrollbar { display:none }`;
|
||||
`nav a { white-space:nowrap; flex-shrink:0 }`;`nav h1 { font-size:0 }`(保留 logo 图、隐藏文字,弥补窄屏空间);
|
||||
`nav > div { flex-shrink:0 }` 右侧语言/主题/退出按钮不压缩。
|
||||
- 顺带在 cmd/gui(electron 客户端)同步了窄屏样式与消息来源徽标(`app.js`/`style.css`,客户端窗口缩放同样受益;
|
||||
客户端需另行构建 electron 应用才生效)。
|
||||
- 验证:部署后 `/` 返回的 dashboard 含 `scrollbar-width:none`/`font-size:0`/`::-webkit-scrollbar` 规则。
|
||||
|
||||
### 步骤 B:删除 core.skills.path 核心配置(已完成)
|
||||
- 删除 `internal/config/registry.go` 两处:`set("core.skills.path", ...)`(SeedDefaults)与
|
||||
`reg(ConfigDef{Key:"core.skills.path", ...})`(定义注册)。
|
||||
- 理由:该键全仓无读取方(grep 仅命中注册处),实际生效路径是 clawhubadapter 的 `skills_dir`
|
||||
(config_clawhubadapter 表 + `core.daemon.data_dir`/skills 兜底)。技能目录属 clawhubadapter 适配领域。
|
||||
- 验证:`grep -rn "skills.path" --include="*.go"` 零命中;部署后设置页无 `core.skills.path`。
|
||||
|
||||
### 步骤 C:清理生产库残留配置(已完成)
|
||||
- 操作前 `sqlite3 .backup /tmp/opencode/config.db.pre-clean.bak`(含微信 token 数据)。
|
||||
- 删除:`config` 表 `channels.wechat.*` 3 键 + `core.skills.path` 键;`DROP TABLE config_wechat /
|
||||
config_openclaw_weixin / config_openclaw`(三者均为历史残留:当前代码零引用,clawhubadapter 实际
|
||||
使用 config_clawhubadapter 表;OC 技能配置在其自身 `~/.openclaw/openclaw.json`)。
|
||||
- 验证:设置页总键数 138→129,无 wechat/weixin/skills.path 残留,`plugin.clawhubadapter.skills_dir /
|
||||
simulator_dir` 正常;服务 healthcheck ready、clawhubadapter "OC plugin manager started"。
|
||||
|
||||
---
|
||||
|
||||
# SDK Stop 注册接口(RegisterStopHandler)计划
|
||||
|
||||
## 一、背景
|
||||
|
||||
2026-08-01 20:00 起生产 homeagent 进入崩溃循环(`fatal error: thread exhaustion`,
|
||||
systemd 重启计数 61+)。排查定位为 SDK 示例插件 `calendar`(示例源码在 SDK 仓库
|
||||
`example/calendar`,生产以 plugin.so 形态加载)三个缺陷叠加:
|
||||
|
||||
1. **农历引擎 3 个 bug**(`daysInLunarYear` 位循环 `i > 0` 应 `i > 0x8`、缺闰月天数、
|
||||
`lunarToSolar` 内层重复加闰月)→ `lunarToSolar(2026,4,12)` 返回 **2062-11-16**(偏移 36 年),
|
||||
农历重复事件(`lunar_yearly`)的 next 被生成到遥远错误日期。
|
||||
2. **`cleanupPastEvents` 保留过时重复事件** → 每 30s ticker 对已到点的重复事件再生成一份 next,
|
||||
事件从 7 个爆炸到 **45612 个**(15MB events.json)。
|
||||
3. **无提醒投递保护**:15018 份同时到点的事件一次性 `go sdk.InjectInterruptText(...)` 投递
|
||||
→ interrupt 风暴 → goroutine/线程耗尽。
|
||||
|
||||
处置:修复农历引擎 3 处 + next 去重 + 清理过时重复事件,用**新版 SDK 仓库 + 新版 plugindev 工具链**
|
||||
重建 `calendar_linux_amd64.hmap`,经 **webui `POST /api/v1/plugins`**(透明代理到 pluginmgr 安装接口)
|
||||
重装,重启验证收敛(事件 4 个、next 正确生成 2027-05-17、0 崩溃)。
|
||||
|
||||
**过程中暴露的能力缺口**:SDK 只有 `Plugin` 接口的 `Name/Start/Stop`,**没有 stop 注册接口**
|
||||
(`RegisterStopHandler`/`OnStop` 均不存在,SDK v0.7.2/v0.8.0/master 一致)。插件停止时只能在自己的
|
||||
`Stop()` 里写清理逻辑,SDK 层无法统一执行"停止时清理"回调;calendar 的 `Stop() { p.saveEvents() }`
|
||||
还会用陈旧内存把已清理的数据写回磁盘(曾导致删除的重复事件复活)。
|
||||
|
||||
## 二、计划
|
||||
|
||||
| # | 动作 | 位置 | 风险 |
|
||||
|---|------|------|------|
|
||||
| 1 | 公共 SDK `PluginSDK` 加 `RegisterStopHandler(fn func())` + `RunStopHandlers()`(幂等、后注册先执行),两处同步 | `third_party/homeagent-sdk/sdk/plugin.go`、SDK 仓库 `sdk/plugin.go` | 低(纯新增,内置 SDK 内嵌透传) |
|
||||
| 2 | 内核 Registry 保存每插件 SDK 引用(`sdkRefs`),`StopAll`/`ReloadOne`/`DisablePlugin` 调 `Stop()` 前执行 `RunStopHandlers` | `internal/plugin/registry.go` | 中(生命周期路径,需回归 reload/disable) |
|
||||
| 3 | 工具链 plugindev:z_bridge 模板 `bridgeState` 存 SDK,`StopPlugin` 先 `RunStopHandlers()` 再 `plugin.Stop()`;init 脚手架模板加演示 | SDK 仓库 `tools/plugindev/templates.go`、`templates/main.go.tmpl` | 低 |
|
||||
| 4 | 内置示例插件演示(如 timer:ticker 停止改为 stop handler) | `internal/plugins/timer/plugin.go` | 低 |
|
||||
| 5 | 外部示例插件同步(`example/calendar` 的 `saveEvents` 改由 stop handler 执行,验证 z_bridge 链路;其余 example 加演示) | SDK 仓库 `example/*` | 低 |
|
||||
| 6 | 文档同步:SDK README 生命周期章节 + 主仓插件开发文档 | SDK 仓库 `README.md`/`README_EN.md` 等 | 无 |
|
||||
|
||||
## 三、实施记录
|
||||
|
||||
1. SDK 公共层(`RegisterStopHandler` + `RunStopHandlers`:后注册先执行、执行后清空幂等)已落地
|
||||
`third_party/homeagent-sdk/sdk/plugin.go`,并同步到 SDK 仓库 `/tmp/opencode/sdk-repo/sdk/plugin.go`(两处一致)。
|
||||
2. 内核 Registry(`internal/plugin/registry.go`)新增 `sdkRefs map[string]*sdk.PluginSDK` + `runStopHandlers`,
|
||||
`loadOne` 注册、`StopAll`/`ReloadOne`/`DisablePlugin` 在 `Stop()` 前执行(共 4 处调用点)。
|
||||
3. 工具链 plugindev(SDK 仓库):linux `tmplLinuxBridge` 的 `go_stop_plugin` 先 `RunStopHandlers()` 再 `plg.Stop()`;
|
||||
windows `tmplBridge` 的 `bridgeState` 加 `sdk` 字段、`StopPlugin` 同链路;`tmplPluginGo` + `main.go.tmpl`
|
||||
脚手架加 `RegisterStopHandler` 演示。plugindev 重新编译通过(GOPATH=/root/go)。
|
||||
4. 内置 timer 插件演示:`close(p.stopCh)` 移入 stop handler,`Stop()` 只 `wg.Wait()`。
|
||||
5. 外部示例:`example/calendar` 的 `saveEvents` 改为 `s.RegisterStopHandler(p.saveEvents)`,
|
||||
`Stop()` 删除写盘调用(持久化交由 stop handler,避免陈旧内存复活已删事件)。
|
||||
6. 文档:SDK 仓库 `README.md`/`README_EN.md` 生命周期章节补充 RegisterStopHandler 说明。
|
||||
7. 构建测试:主仓 `go build ./...` + `go test ./internal/sdk/... ./internal/plugin/...` 全绿;
|
||||
SDK 仓库 `go build ./...` + `go test ./sdk/...` 全绿。
|
||||
8. 生产部署验证:新 plugindev(--no-bundle)重建 `calendar_linux_amd64.hmap`(md5 084e97c0…,strings 确认
|
||||
`go_stop_plugin → RunStopHandlers → saveEvents` 编译进 plugin.so);webui API 删旧装新;重装新内核
|
||||
homed(含 sdkRefs/runStopHandlers);两次重启事件稳定 3 个不复活、events.json mtime 与 stop 时刻吻合
|
||||
(saveEvents 经 stop handler 真实执行)、0 次 thread exhaustion、服务 active。
|
||||
|
||||
|
||||
---
|
||||
|
||||
# clawhubadapter OpenClaw 通道插件兼容修复计划
|
||||
|
||||
## 一、背景
|
||||
|
||||
生产 `core.llm.provider` 已是 mock LLM 源(`core.llm.sources.mocktest`,base_url
|
||||
`http://127.0.0.1:18080/v1`、model mock-model、adapter openai),mock LLM 服务常驻运行。
|
||||
借助 **mock 通道插件**(`/tmp/opencode/mock-skills/mock-wechat/`,完全复刻 openclaw-weixin 的
|
||||
真实注册格式 `register(api) → api.registerChannel({ plugin: ChannelPlugin })`)放入生产 skills 目录
|
||||
端到端复现,得出如下结论:
|
||||
|
||||
**已验证可用链路**:manager 加载 mock 插件 → Go 端识别 `ocplugin mock-wechat handled by manager` →
|
||||
注册工具 `mock-wechat_read_mock_wechat_input`/`mock-wechat_mock_echo` → `RegisterOutputChannel("mock-wechat")`
|
||||
→ mock 自推消息经 `channel_input` 通知 → `[agent] interrupt from manager/mock-wechat` →
|
||||
mock LLM 正常回复(195ms)。
|
||||
|
||||
**复现的核心缺陷**(真实通道插件 wechat/dingding"根本不可用"的根因):
|
||||
|
||||
1. **输出断链**:manager `tools/call` 通道分支只认 `channelPlugin.outbound.sendText/sendMedia`
|
||||
(旧格式),真实 ChannelPlugin(openclaw-weixin 等)无 outbound →
|
||||
`tools/call mock-wechat → error: "channel mock-wechat has no output handler"`。
|
||||
2. **生命周期静止**:manager mock api 从不调用 `gateway.startAccount/stopAccount`,也无
|
||||
`api.runtime`/`channelRuntime` → 通道插件加载后永不启动(不登录、不轮询、不收消息)。
|
||||
3. **输入依赖错位**:真实插件把消息经 `channelRuntime.reply.dispatchReplyWithBufferedBlockDispatcher`
|
||||
推送(manager 完全无此对象),而不是调 `api.submitInput`。
|
||||
4. **stdout 污染**:插件 `console.log` 直接进 JSON-RPC 流,Go 端 readLoop 跳过非 JSON 行,有丢通知风险。
|
||||
|
||||
**wechat 通道的心跳机制**(`openclaw-weixin/dist/index.js` `pollLoop`,448 行起):每账号一个常驻
|
||||
`pollLoop`,循环 `POST ilink/bot/getupdates`(body `{get_updates_buf}`,超时 35s)——**长轮询即心跳**:
|
||||
服务器收到 poll 请求即知通道在线,新消息随 poll 响应 push 回来;超时视为空响应继续轮询,真错误延时
|
||||
5s 重试。**与 gateway 生命周期强绑定**:
|
||||
|
||||
- `pollLoop` 只由 `gateway.startAccount(ctx)` 启动;不被调用 → 心跳/收消息全断(服务器侧认为通道离线)。
|
||||
- `startAccount` 末尾 `await new Promise(()=>{})` **永久挂起**——OC gateway 靠它配合 health-monitor
|
||||
(startAccount 退出 → 判定账号崩溃 → 重启账号)。manager 调 `startAccount` 必须 **fire-and-forget**。
|
||||
- 停靠 `gateway.stopAccount(ctx)`(`ctx.account.accountId` 定位),停止时经 `statusSinks` 调
|
||||
`ctx.setStatus({running:false, connected:false, lastStopAt})`;启动即上报
|
||||
`ctx.getStatus()/ctx.setStatus({...running:true, connected:true, lastStartAt})`——`connected` 是
|
||||
gateway 判断账号存活的依据。
|
||||
- `sendTyping`(ilink/bot/sendtyping + typing_ticket)是打字指示,非心跳,无需支持。
|
||||
|
||||
## 二、修复计划
|
||||
|
||||
| # | 动作 | 位置 | 风险 |
|
||||
|---|------|------|------|
|
||||
| A | manager 提供 **gateway 生命周期桥**:channel 插件注册后自动 `gateway.startAccount(ctx)`(fire-and-forget,不等待挂起的 Promise),构造完整 ctx `{account, cfg, channelRuntime, getStatus, setStatus}`;Go 端 `channel_stop` 通知 → `stopAccount` | `internal/plugins/clawhubadapter/manager/main.js` | 中 |
|
||||
| B | 实现 **channelRuntime mock**:`reply.dispatchReplyWithBufferedBlockDispatcher`(deliver 回调 → `channel_output` 通知送 Go 端)、`getPolls`(OC 通用通道轮询输入)、`call` 透传 | 同上 | 中 |
|
||||
| C | `tools/call` 通道分支改造:无 `outbound` 的 ChannelPlugin 改走 channelRuntime 事件式发送(agent 输出 → deliver),不再报 "no output handler" | 同上 | 低 |
|
||||
| D | 状态上报透传:`setStatus` 经 `channel_status` 通知 → Go 端可查;health-monitor 语义(startAccount 保持挂起) | 同上 | 低 |
|
||||
| E | stdout 卫生:插件 `console.log` 重定向 stderr(或 JSON-RPC 流感知封装),杜绝污染 | 同上 | 低 |
|
||||
| F | Go 端:`channel_output`/`channel_status` 通知接入(事件分发),通道输出 handler 保持 `sp.CallTool` | `internal/plugins/clawhubadapter/registry.go`、`plugin.go` | 中 |
|
||||
| G | 端到端验证:mock 通道插件 + mock LLM(生产环境,临时放入/移出 skills 目录)复跑全链路(登录启动→收消息→回复→出站→停止) | 生产 | 低 |
|
||||
|
||||
## 三、实施记录
|
||||
|
||||
(逐步填写)
|
||||
|
||||
1. **manager/main.js — 通道运行时与生命周期桥(已完成,独立运行验证)**
|
||||
- `makeChannelRuntime(chName, ch)`:`reply.dispatchReplyWithBufferedBlockDispatcher(opts)` 提取
|
||||
`dispatcherOptions.deliver`/`typingCallbacks` 按 `ctx.AccountId` 挂到 `ch.deliverers`,随后
|
||||
`notify('channel_input', {channel, payload:{content: BodyForAgent||Body, from, sessionKey, accountId,
|
||||
messageSid, chatType, raw}})` 入站;返回 dispatcher(sendNow/addToBuffer/sendBuffer/closeBuffer)。
|
||||
`chatPolls`/`getPolls` 空转(防断连误判)、`call` 转发 `channel_output` 通知。
|
||||
- `startChannels(name)`:channel 插件注册后自动枚举账号(`config.listAccountIds`→`resolveAccount`,
|
||||
缺省 `['default']`),构造完整 ctx `{account, cfg, channelRuntime, getStatus, setStatus}`,
|
||||
**fire-and-forget** 调 `gateway.startAccount`(真实插件会永久挂起,绝不等待);崩溃/状态变更经
|
||||
`channel_status` 通知透传(health-monitor 语义:startAccount 不退出=账号存活)。
|
||||
- `stopChannels(name)`:逐个账号 `gateway.stopAccount`;进程 SIGTERM/SIGINT 时统一执行优雅停靠。
|
||||
- `tools/call` 通道分支:保留 outbound(旧格式)→ 新增 **deliver 事件式发送**
|
||||
(`deliverItem` 按 accountId 取 deliver + typingCallbacks.onReplyStart/onCleanup 包裹)→
|
||||
无 deliver 时降级 `channel_output` 通知 → 兜底报错。不再出现 "has no output handler"。
|
||||
- **stdout 卫生**:全局 `console.log` 重定向 stderr,JSON-RPC 流仅承载协议帧。
|
||||
2. **mock 插件升级(/tmp/opencode/mock-skills/mock-wechat/index.js)**:完全复刻真实 weixin 行为——
|
||||
`gateway.startAccount` 永久挂起 + `setStatus` 上报 + `setInterval` 心跳轮询 + 800ms 后经
|
||||
`dispatchReplyWithBufferedBlockDispatcher` 推送入站(deliver 本地记录发送);`stopAccount` 停轮询+状态置否。
|
||||
3. **manager 独立运行验证(通过)**:`channel_status` 启动上报(running=true connected=true);
|
||||
`tools/call mock-wechat` → `{"status":"sent","via":"channelRuntime.deliver"}`,插件 deliver 收到
|
||||
`text="hello from agent"` 且 typing onReplyStart/onCleanup 正确包裹;心跳 poll #1-4 常驻;
|
||||
SIGTERM → `stopAccount called` 退出码 0;插件 console 输出全部走 stderr(协议流零污染)。
|
||||
4. **Go 端通知接入(plugin.go translateAndRegister + registry.go 状态缓存)**:`channel_status` 存
|
||||
`channelStatus` map(可查)+ 日志;`channel_output` 降级事件日志。`go build ./...` 通过。
|
||||
5. **回复闭环修复(同步注入)**:`channel_input → InjectInterruptText` 的 InputEvent 不带 ResponseCh
|
||||
(internal/agent/io/channel.go:276),agent 回复在 emitResponse(eventloop.go:384)被静默丢弃。
|
||||
改为 `s.InjectInputSync(pluginName, channel, "text", payload)`(内部 SDK 已有 4 参版本,返回
|
||||
`*OutputEvent`)同步等待回复 → 提取 `Payload["content"]` → `sp.CallTool(channel, {payload, meta})`
|
||||
→ manager `tools/call` → deliver → 插件发送 → 微信送达。公共 SDK IOInjector 同步补
|
||||
`InjectInputSync(source, channel, text) string`(ioAdapter 实现,供外部插件一致使用)。
|
||||
6. **mock LLM 恒定文本化**:/opt/llm-mock/mock_server.py `decide()` 删除工具调用分支,一律回文本
|
||||
("无论收到什么消息都通过微信插件发送"),保证每条入站消息回复必然走通道输出。
|
||||
7. **生产微信闭环验证(通过)**:用户微信发"你好..." → pollLoop 收到 → dispatchReply →
|
||||
InjectInputSync → mock LLM 回文本 → CallTool(wechat) → deliver → `POST ilink/bot/sendmessage`
|
||||
→ **status=200 message_id=7489545365740590088**,微信收到"(mock)已收到消息,长度 324 字符。"
|
||||
8. **通用性审查(无硬编码)**:manager/plugin.go/registry.go 均无 weixin/wechat 特判,全部按 OC 规范
|
||||
字段实现(gateway/config/capabilities/channelRuntime/deliver/typingCallbacks)。修正规范签名参数
|
||||
约定:`listAccountIds(cfg)`、`resolveAccount(cfg, accountId)` 正确传 cfg。
|
||||
9. **已知边界**(非硬编码,架构性):a) `channelRuntime.getPolls/chatPolls` 返回空 msgs——依赖
|
||||
runtime 轮询输入的通用通道型插件收不到消息(weixin/dingding 类自带 pollLoop 的通道不受影响);
|
||||
b) `gatewayMethods` 登录流程(web.login.start/QR 扫码)未实现(CLI 有 stub),通道凭 token
|
||||
配置直连;c) startAccount ctx 提供 account/channelRuntime/cfg/getStatus/setStatus 核心字段。
|
||||
10. **补充边界 a) getPolls/chatPolls 消息源(已完成)**:manager `makeChannelRuntime` 的
|
||||
`chatPolls/getPolls` 改为读 `ch.pollQueues`(按 accountId 队列,poll 取走即消费);新增
|
||||
`channel/send` RPC(Go 端注入 → 队列 → 插件轮询取走);Go 端 `SendToChannel(channel, payload)`
|
||||
+ `ChannelSender()` 单例(Start 时置位)。验证:mock-poll 插件(纯 chatPolls 轮询型)——
|
||||
`channel/send` → `{"status":"queued"}` → `[mock-poll] poll got msg` → dispatchReply →
|
||||
`channel_input` 入站完整(content/from/sessionKey/accountId/messageSid/chatType)。
|
||||
微信链路回归正常(Polling started + channel_status running=true)。
|
||||
11. **边界 b) 登录流程核实(已解决,无需实现)**:真实登录机制是 **SKILL 脚本旁路**——
|
||||
`weixin-openclaw-login` SKILL 的 `scripts/get-login-url.js`(ilink 二维码 URL)+
|
||||
`poll-login-status.py`(轮询扫码状态)→ agent 经 exec 执行 → 拿 bot_token 写入
|
||||
`~/.openclaw-weixin/account.json`(2026-07-28 17:31 创建,token 有效)→ 插件启动
|
||||
`resolveAccountData` 直读。不依赖 manager gatewayMethods(web.login.start 等 OC gateway
|
||||
协议 stub 不影响真实使用)。
|
||||
12. **clawhubadapter 全量管理接口(已完成并验证)**:向 agent 暴露完整插件/通道管理面——
|
||||
- 新工具:`clawhubadapter_plugin_info`(类型/工具/关联通道详情)、`plugin_reload`(reloadPlugin)、
|
||||
`channel_list`(全部注册通道 + 实时状态)、`channel_send`(SendToChannel 投递)、
|
||||
`channel_start`/`channel_stop`(manager `channel/start`、`channel/stop` RPC)。
|
||||
- manager 新增 RPC:`plugins/channels`(registeredChannels 摘要含 status/accounts)、
|
||||
`channel/start`(fire-and-forget startChannels 恢复账号)、`channel/stop`(stopAccount,
|
||||
按 channel 全停或按 accountId 单停,省略 channel 则全部停止)。
|
||||
- Go 侧 `channelSummary(mgr)` 合并 manager 注册信息与 `channel_status` 实时缓存(缓存优先)。
|
||||
- 端到端验证(生产实例,LLM 为本地 mock 源):微信发"你好通道..." → agent 执行
|
||||
`channel_list` → `- wechat | plugin=openclaw-weixin type=text running=true connected=true
|
||||
accounts=[default]` → 回复送达;发"注入..." → 执行 `channel_send` → "消息已投递到通道
|
||||
wechat" → 回复送达。standalone manager 另验证 `channel/start`(mock-wechat startAccount
|
||||
重新执行、心跳恢复)与 `channel/stop`(stopAccount called)。
|
||||
13. **插件删除回调 onRemove 全套(已完成并验证)**:`RegisterOnRemoveHandler`(仅卸载触发、
|
||||
重载/禁用不触发,与 stop handler 互补——stop 每次停止都执行)。registry.RemovePlugin 流程:
|
||||
stop handlers → Stop → runOnRemoveHandlers → 移除 plugins/sdkRefs/instances →
|
||||
UnregisterPluginTools → **配置清理**。配置清理含两层:`ConfigRegistry.RemovePlugin`
|
||||
删除 defs 中 `plugin.<name>.*` 配置项定义 + DROP `config_<name>` 插件配置表
|
||||
(含用户设置值,ListPlugins 基于 config_% 表枚举故配置区完全消失);已用临时程序验证
|
||||
(before: defs=1/plugins=[timer] → after: defs=0/plugins=[])。示例盘点(SDK 仓库 15 个):
|
||||
calendar(events.json)、memo(memos.json)、rss(订阅数据目录)、weather(缓存目录)已加;
|
||||
files(filesDir 为用户配置的访问根目录,默认 /)、bili/qq(用户下载资产)、
|
||||
ocr(函数内 defer RemoveAll 自清理)按语义不加;plugindev 模板 main.go.tmpl + README.tmpl
|
||||
含 onRemove 演示;SDK README/README_EN 生命周期文档补"删除清理(onRemove)"小节。
|
||||
|
||||
14. [2026-08-03] SDK 工具链/打包/重装 + dlclose 修复:
|
||||
- 工具链源码位置澄清:SDK 仓完整内容位于 third_party/homeagent-sdk(主仓 .gitignore 仅跟踪
|
||||
sdk/meta/go.mod,"两个远程仓库各取所需";/tmp/opencode/sdk-repo 为工作克隆,远程=gitcode)。
|
||||
- 工具链支持公共 IOInjector.InjectInputSync:CORE_INJECT_INPUT_SYNC=47(C 桥 dispatchIO
|
||||
callString 回传回复文本);主仓 cabi loader case 47 用内部 4 参版 InjectInputSync 取
|
||||
OutputEvent.Payload["content"] setResult(meta.go ID 47 + loader.go,主仓 3f252ed)。
|
||||
- plugindev 构建环境:GOMODCACHE=/root/go/pkg/mod(yaegi 缓存所在)、GOPROXY=off。
|
||||
- 工具链打包 memo:plg.json BOM 去除、name_en "Memo/Notes"→"Memo"(toSnake 不处理斜杠,
|
||||
name_en 带 / 会使 hmap 名含子路径报错);bundle=true 时走全平台交叉编译(本机无 darwin
|
||||
工具链),打包用 `build --no-bundle --target linux/amd64`;产物 dist/memo_linux_amd64.hmap。
|
||||
- 重装:pluginmgr HTTP API(127.0.0.1:9876)DELETE /plugins/memo 卸载(走内核 RemovePlugin
|
||||
+ onRemove)→ POST /plugins binary body 传 hmap(返回 installed+checksum);生效用
|
||||
webui `POST /api/v1/plugins/reload`(X-API-Key,生产 admin123)。
|
||||
- 关键 bug:Linux dlopen 同路径复用旧句柄——RemovePlugin/ReloadOne 只 Stop 不 dlclose,
|
||||
插件二进制更新后重载仍执行旧代码(生产 memo 装新版仍注册旧 3 工具)。修复:
|
||||
cabiPlugin.Close()(handle.Close)+ Registry.closeDynamic 在卸载/重载时调用(主仓 649e312)。
|
||||
生产 homed-new7 验证:memo 6 工具(memo_todo_add/complete/list + memo_memo_create/list/delete)
|
||||
注册正常,wechat 通道 running。
|
||||
- SDK 仓推送 b6e30f9(工具链 47 + 重建 bin 二进制 + memo plg.json + sdk/plugin.go 注释精简)。
|
||||
15. [2026-08-03] 嵌套 git 恢复 + 工作区清理 + codegraph 索引修正:
|
||||
- 嵌套 git 恢复:third_party/homeagent-sdk 原本是"单仓库双提交"(目录内嵌套 .git 推 gitcode
|
||||
homeagent-sdk 仓,主仓 git 跟踪 sdk/meta/go.mod 推 HomeAgent 仓),嵌套 .git 此前被误删;
|
||||
已从 /tmp/opencode/sdk-repo 复制 .git 恢复(remote=homeagent-sdk.git,HEAD=b6e30f9,工作区干净),
|
||||
主仓 git 不受影响。以后 SDK 改动直接在 third_party 内 git commit+push(SDK 仓推送仍用带凭据
|
||||
URL https://JianFeeeee:BCkb32xBuLxWD9P4MmU8ydZ5@gitcode.com/JianFeeeee/homeagent-sdk.git),
|
||||
不再经 /tmp 中转。
|
||||
- /tmp 清理:删除 /tmp/opencode/sdk-repo、hasdk-fresh、plugindev-new、plugindev_new、mock-run、
|
||||
lunartest(SDK 中转/临时目录);保留 homed-new*(生产二进制备份)、mock-skills 等非 SDK 内容。
|
||||
- replace 修正:go.mod 第 17 行已是 `./third_party/homeagent-sdk`(正确);package-linux.sh
|
||||
prepare_gomod 优先用 $PROJECT_ROOT/third_party/homeagent-sdk,仅缺失时才 clone /tmp/homeagent-sdk
|
||||
兜底(主仓 108faac)。
|
||||
- codegraph 索引修正:根目录 codegraph.json(PROJECT_CONFIG_FILENAME)配
|
||||
includeIgnored+include: ["third_party/homeagent-sdk"],codegraph index 后 Files 179→233,
|
||||
third_party 文件 7→61,tools/plugindev 与 sdk/plugin.go(InjectInputSync 等)均可查询
|
||||
(此前嵌套 SDK 仓被主仓 .gitignore 挡在索引外);codegraph sync 不感知配置变更,需 index 全量重建。
|
||||
@ -2,7 +2,10 @@ package config
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
@ -28,10 +31,11 @@ type ConfigDef struct {
|
||||
}
|
||||
|
||||
type ConfigRegistry struct {
|
||||
mu sync.RWMutex
|
||||
db *sql.DB
|
||||
dbPath string
|
||||
defs map[string]*ConfigDef
|
||||
mu sync.RWMutex
|
||||
db *sql.DB
|
||||
dbPath string
|
||||
defs map[string]*ConfigDef
|
||||
llmSnapFile string
|
||||
}
|
||||
|
||||
func NewConfigRegistry(dbPath string) *ConfigRegistry {
|
||||
@ -265,6 +269,9 @@ func (r *ConfigRegistry) Get(key string) (interface{}, error) {
|
||||
func (r *ConfigRegistry) Set(key string, value interface{}) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if strings.HasPrefix(key, "core.llm.") {
|
||||
r.writeLLMSnapshotLocked()
|
||||
}
|
||||
_, err := r.db.Exec(`INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)`, key, fmt.Sprint(value))
|
||||
if err == nil && strings.HasPrefix(key, "core.llm.sources.") {
|
||||
rest := strings.TrimPrefix(key, "core.llm.sources.")
|
||||
@ -303,6 +310,107 @@ func (r *ConfigRegistry) Delete(key string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// SnapshotCoreLLM 捕获全部 core.llm.* 键值(LLM 源密度快照),供写前留档。
|
||||
// 返回值是 key→value 的不可变拷贝;写入 guard 配置恢复的基线。
|
||||
func (r *ConfigRegistry) SnapshotCoreLLM() map[string]string {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
return r.listPrefixLocked("core.llm.")
|
||||
}
|
||||
|
||||
// SetLLMSnapshotFile 设定写前留档文件:此后任意写入 core.llm.* 键时,
|
||||
// 先把当前 llm 配置整体快照到该文件(guard 恢复的外部基线)。
|
||||
func (r *ConfigRegistry) SetLLMSnapshotFile(path string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if path != "" {
|
||||
r.writeLLMSnapshotToFileLocked(path)
|
||||
}
|
||||
r.llmSnapFile = path
|
||||
}
|
||||
|
||||
// writeLLMSnapshotLocked 调用方须持有写锁;若已配置快照文件则写入当前 llm 快照。
|
||||
func (r *ConfigRegistry) writeLLMSnapshotLocked() {
|
||||
if r.llmSnapFile == "" {
|
||||
return
|
||||
}
|
||||
r.writeLLMSnapshotToFileLocked(r.llmSnapFile)
|
||||
}
|
||||
|
||||
func (r *ConfigRegistry) writeLLMSnapshotToFileLocked(path string) {
|
||||
snap := r.listPrefixLocked("core.llm.")
|
||||
if err := SaveLLMSnapshot(path, snap); err != nil {
|
||||
log.Printf("[config] save llm snapshot %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RestoreCoreLLM 精确还原到快照状态:快照里有的键回写旧值,
|
||||
// 当前存在但快照里没有的 core.llm.* 键删除(保持与快照一致)。
|
||||
func (r *ConfigRegistry) RestoreCoreLLM(snap map[string]string) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
current := r.listPrefixLocked("core.llm.")
|
||||
seen := make(map[string]bool, len(snap))
|
||||
for k, v := range snap {
|
||||
seen[k] = true
|
||||
if _, err := r.db.Exec(`INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)`, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for k := range current {
|
||||
if seen[k] {
|
||||
continue
|
||||
}
|
||||
if _, err := r.db.Exec(`DELETE FROM config WHERE key = ?`, k); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SaveLLMSnapshot 把 LLM 快照持久化到文件(guard 恢复的外部基线)。
|
||||
func SaveLLMSnapshot(path string, snap map[string]string) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(snap, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
// LoadLLMSnapshot 从文件读回 LLM 快照。
|
||||
func LoadLLMSnapshot(path string) (map[string]string, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
snap := make(map[string]string)
|
||||
if err := json.Unmarshal(data, &snap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return snap, nil
|
||||
}
|
||||
|
||||
// listPrefixLocked 调用方须持有锁;返回 prefix 开头的全部键值。
|
||||
func (r *ConfigRegistry) listPrefixLocked(prefix string) map[string]string {
|
||||
out := make(map[string]string)
|
||||
rows, err := r.db.Query(`SELECT key, value FROM config WHERE key LIKE ? ORDER BY key`, prefix+"%")
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var k, v string
|
||||
if err := rows.Scan(&k, &v); err == nil {
|
||||
out[k] = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (r *ConfigRegistry) Dump() map[string]interface{} {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
@ -464,6 +572,7 @@ func (r *ConfigRegistry) seedCoreDefs(dataDir string) {
|
||||
reg(ConfigDef{Key: "core.daemon.heartbeat_interval", Default: "15s", Type: "duration", DisplayName: "心跳间隔", Description: "Agent 心跳检查间隔", Category: "daemon"})
|
||||
reg(ConfigDef{Key: "core.daemon.check_interval", Default: "30s", Type: "duration", DisplayName: "检查间隔", Description: "网络状态检查间隔", Category: "daemon"})
|
||||
reg(ConfigDef{Key: "core.daemon.log_level", Default: "info", Type: "select", DisplayName: "日志级别", Description: "日志输出级别", Options: []string{"debug", "info", "warn", "error"}, Category: "daemon"})
|
||||
reg(ConfigDef{Key: "core.defaults.llm_endpoints", Default: "", Type: "string", DisplayName: "探活端点", Description: "健康检查的 LLM 探活端点,逗号分隔;留空自动取 LLM 源 base_url", Category: "daemon"})
|
||||
|
||||
reg(ConfigDef{Key: "core.llm.provider", Default: "deepseek", Type: "string", DisplayName: "默认提供商", Description: "默认 LLM 提供商名称,需匹配 sources 中的定义", Category: "llm"})
|
||||
reg(ConfigDef{Key: "core.llm.model", Default: "deepseek-v4-flash", Type: "string", DisplayName: "默认模型", Description: "默认 LLM 模型名称", Category: "llm"})
|
||||
@ -698,6 +807,23 @@ func (r *ConfigRegistry) ToConfig() *types.Config {
|
||||
|
||||
cfg.Plugin.Dir = read("core.plugin.dir", cfg.Plugin.Dir)
|
||||
|
||||
// 探活端点:优先显式配置,缺省取 LLM 源 base_url(去重),保证健康检查有实际目标
|
||||
if eps := read("core.defaults.llm_endpoints", ""); eps != "" {
|
||||
for _, ep := range strings.Split(eps, ",") {
|
||||
if ep = strings.TrimSpace(ep); ep != "" {
|
||||
cfg.Defaults.LLMEndpoints = append(cfg.Defaults.LLMEndpoints, ep)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
seen := make(map[string]bool)
|
||||
for _, src := range cfg.LLM.Sources {
|
||||
if src.BaseURL != "" && !seen[src.BaseURL] {
|
||||
seen[src.BaseURL] = true
|
||||
cfg.Defaults.LLMEndpoints = append(cfg.Defaults.LLMEndpoints, src.BaseURL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cfg.InputProcessing.Image.FallbackProvider = read("core.input_processing.image.fallback_provider", cfg.InputProcessing.Image.FallbackProvider)
|
||||
cfg.InputProcessing.Image.FallbackModel = read("core.input_processing.image.fallback_model", cfg.InputProcessing.Image.FallbackModel)
|
||||
cfg.InputProcessing.Image.DescribePrompt = read("core.input_processing.image.describe_prompt", cfg.InputProcessing.Image.DescribePrompt)
|
||||
|
||||
@ -220,3 +220,73 @@ func TestGetHelpers(t *testing.T) {
|
||||
t.Fatalf("GetBool fallback: expected true, got %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotRestoreCoreLLM(t *testing.T) {
|
||||
r := NewConfigRegistry("")
|
||||
defer r.Close()
|
||||
|
||||
r.Set("core.llm.sources.main.base_url", "https://a")
|
||||
r.Set("core.llm.sources.main.model", "m1")
|
||||
r.Set("core.llm.sources.main.api_key", "k1")
|
||||
|
||||
snap := r.SnapshotCoreLLM()
|
||||
if len(snap) != 3 {
|
||||
t.Fatalf("expected 3 keys, got %d: %v", len(snap), snap)
|
||||
}
|
||||
|
||||
// 模拟写坏
|
||||
r.Set("core.llm.sources.main.base_url", "https://broken")
|
||||
r.Set("core.llm.sources.main.api_key", "hacked")
|
||||
r.Set("core.llm.sources.extra.model", "intruder")
|
||||
|
||||
if err := r.RestoreCoreLLM(snap); err != nil {
|
||||
t.Fatalf("RestoreCoreLLM: %v", err)
|
||||
}
|
||||
got := r.SnapshotCoreLLM()
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("after restore expected 3 keys, got %d: %v", len(got), got)
|
||||
}
|
||||
for k, v := range snap {
|
||||
if got[k] != v {
|
||||
t.Errorf("key %s: want %q got %q", k, v, got[k])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMSnapshotFile(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "llm_snapshot.json")
|
||||
snap := map[string]string{"core.llm.sources.main.base_url": "https://a", "core.llm.sources.main.model": "m1"}
|
||||
if err := SaveLLMSnapshot(path, snap); err != nil {
|
||||
t.Fatalf("Save: %v", err)
|
||||
}
|
||||
got, err := LoadLLMSnapshot(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if got["core.llm.sources.main.base_url"] != "https://a" || got["core.llm.sources.main.model"] != "m1" {
|
||||
t.Fatalf("round-trip mismatch: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetLLMSnapshotFile(t *testing.T) {
|
||||
r := NewConfigRegistry("")
|
||||
defer r.Close()
|
||||
r.Set("core.llm.sources.main.base_url", "https://orig")
|
||||
r.Set("core.llm.sources.main.model", "m0")
|
||||
|
||||
path := filepath.Join(t.TempDir(), "llm_pre.json")
|
||||
r.SetLLMSnapshotFile(path)
|
||||
// 再次写入:写前自动留档应记录当前值 orig/m0,随后才被覆盖
|
||||
r.Set("core.llm.sources.main.base_url", "https://broken")
|
||||
|
||||
got, err := LoadLLMSnapshot(path)
|
||||
if err != nil {
|
||||
t.Fatalf("Load snapshot: %v", err)
|
||||
}
|
||||
if got["core.llm.sources.main.base_url"] != "https://orig" {
|
||||
t.Fatalf("write-ahead snapshot should record pre-write value, got %q", got["core.llm.sources.main.base_url"])
|
||||
}
|
||||
if got["core.llm.sources.main.model"] != "m0" {
|
||||
t.Fatalf("snapshot missing untouched key model: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
143
internal/ipc/ipc.go
Normal file
143
internal/ipc/ipc.go
Normal file
@ -0,0 +1,143 @@
|
||||
// Package ipc 实现 guard↔worker 之间的 PING/ACK 心跳协议:worker 监听 unix socket,
|
||||
// guard 发 PING,worker 回 ACK(含 kernel 状态快照),实现带自诊断上报的存活判定,
|
||||
// 取代单纯的文件心跳 + 退出码。
|
||||
package ipc
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// sinkPath 心跳 socket 文件路径。
|
||||
func sinkPath(dataDir string) string {
|
||||
return filepath.Join(dataDir, "worker.ipc")
|
||||
}
|
||||
|
||||
// Status worker 上报给 guard 的自诊断快照。
|
||||
type Status struct {
|
||||
PID int `json:"pid"`
|
||||
Boot string `json:"boot"` // normal | failback
|
||||
UptimeSec int64 `json:"uptime_sec"` // 进程存活秒数
|
||||
LLMOK *bool `json:"llm_ok,omitempty"` // rescue 源可达性(nil=未探)
|
||||
Tools int `json:"tools"` // 已注册工具数
|
||||
LastDiag string `json:"last_diag,omitempty"` // 最近一次自诊断结论(如 diag_loc.cause)
|
||||
}
|
||||
|
||||
// StatusFunc 组装 worker 当前状态。
|
||||
type StatusFunc func() *Status
|
||||
|
||||
// Server worker 侧:监听 unix socket,处理 PING→ACK。
|
||||
type Server struct {
|
||||
path string
|
||||
ln net.Listener
|
||||
status StatusFunc
|
||||
stop chan struct{}
|
||||
done chan struct{}
|
||||
}
|
||||
|
||||
// NewServer 创建心跳服务端(尚未 Listen,见 Start)。
|
||||
func NewServer(dataDir string, status StatusFunc) *Server {
|
||||
return &Server{path: sinkPath(dataDir), status: status, stop: make(chan struct{}), done: make(chan struct{})}
|
||||
}
|
||||
|
||||
// Start 启动监听。若 socket 已存在则先清理(无心跳残留)。
|
||||
func (s *Server) Start() error {
|
||||
if s.status == nil {
|
||||
s.status = func() *Status { return nil }
|
||||
}
|
||||
if err := os.RemoveAll(s.path); err != nil {
|
||||
return err
|
||||
}
|
||||
ln, err := net.Listen("unix", s.path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ipc listen %s: %w", s.path, err)
|
||||
}
|
||||
s.ln = ln
|
||||
go s.acceptLoop()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) acceptLoop() {
|
||||
defer close(s.done)
|
||||
for {
|
||||
conn, err := s.ln.Accept()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
go s.handle(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handle(conn net.Conn) {
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
sc := bufio.NewScanner(conn)
|
||||
if !sc.Scan() {
|
||||
return
|
||||
}
|
||||
line := sc.Text()
|
||||
if line != "PING" {
|
||||
return
|
||||
}
|
||||
resp := map[string]interface{}{"type": "ACK"}
|
||||
if st := s.status(); st != nil {
|
||||
resp["status"] = st
|
||||
}
|
||||
data, _ := json.Marshal(resp)
|
||||
conn.Write(append(data, '\n'))
|
||||
}
|
||||
|
||||
// Stop 关闭监听。
|
||||
func (s *Server) Stop() {
|
||||
if s.ln != nil {
|
||||
s.ln.Close()
|
||||
}
|
||||
<-s.done
|
||||
}
|
||||
|
||||
// Client guard 侧:向 worker 心跳 socket 发 PING 并等 ACK。
|
||||
type Client struct {
|
||||
path string
|
||||
}
|
||||
|
||||
// NewClient 创建客户端。
|
||||
func NewClient(dataDir string) *Client {
|
||||
return &Client{path: sinkPath(dataDir)}
|
||||
}
|
||||
|
||||
// Ping 发一次 PING、收 ACK。timeout 内未收到返回错误。返回 (status, error);
|
||||
// status 可能为 nil(ACK 无状态体)。
|
||||
func (c *Client) Ping(timeout time.Duration) (*Status, error) {
|
||||
conn, err := net.DialTimeout("unix", c.path, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(timeout))
|
||||
if _, err := conn.Write([]byte("PING\n")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sc := bufio.NewScanner(conn)
|
||||
if !sc.Scan() {
|
||||
return nil, fmt.Errorf("ipc: empty ACK")
|
||||
}
|
||||
var resp struct {
|
||||
Type string `json:"type"`
|
||||
Status *Status `json:"status"`
|
||||
}
|
||||
if err := json.Unmarshal(sc.Bytes(), &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Type != "ACK" {
|
||||
return nil, fmt.Errorf("ipc: unexpected reply %q", resp.Type)
|
||||
}
|
||||
return resp.Status, nil
|
||||
}
|
||||
|
||||
// Close 无状态(客户端用完即释放连接)。
|
||||
func (c *Client) Close() {}
|
||||
67
internal/ipc/ipc_test.go
Normal file
67
internal/ipc/ipc_test.go
Normal file
@ -0,0 +1,67 @@
|
||||
package ipc
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestPingAck(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ok := true
|
||||
srv := NewServer(dir, func() *Status {
|
||||
return &Status{PID: 123, Boot: "normal", UptimeSec: 42, LLMOK: &ok, Tools: 5, LastDiag: "diag:network"}
|
||||
})
|
||||
if err := srv.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
defer srv.Stop()
|
||||
|
||||
cli := NewClient(dir)
|
||||
st, err := cli.Ping(2 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
if st == nil {
|
||||
t.Fatal("nil status")
|
||||
}
|
||||
if st.PID != 123 || st.Boot != "normal" || st.Tools != 5 {
|
||||
t.Fatalf("status mismatch: %+v", st)
|
||||
}
|
||||
if st.LLMOK == nil || !*st.LLMOK {
|
||||
t.Fatal("LLMOK should be true")
|
||||
}
|
||||
if st.LastDiag != "diag:network" {
|
||||
t.Fatalf("LastDiag = %q", st.LastDiag)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPingTimeout(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// 未启动 server → 立即超时
|
||||
cli := NewClient(dir)
|
||||
start := time.Now()
|
||||
_, err := cli.Ping(500 * time.Millisecond)
|
||||
if err == nil {
|
||||
t.Fatal("expected error when no server")
|
||||
}
|
||||
if time.Since(start) > 3*time.Second {
|
||||
t.Fatal("timeout took too long")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerNoStatusFunc(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
srv := NewServer(dir, nil)
|
||||
if err := srv.Start(); err != nil {
|
||||
t.Fatalf("Start: %v", err)
|
||||
}
|
||||
defer srv.Stop()
|
||||
cli := NewClient(dir)
|
||||
st, err := cli.Ping(2 * time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("Ping: %v", err)
|
||||
}
|
||||
if st != nil {
|
||||
t.Fatalf("expected nil status when no status func, got %+v", st)
|
||||
}
|
||||
}
|
||||
@ -44,6 +44,7 @@ func NewMonitor(interval time.Duration) *Monitor {
|
||||
},
|
||||
},
|
||||
interval: interval,
|
||||
status: make([]EndpointStatus, 0),
|
||||
}
|
||||
}
|
||||
|
||||
@ -143,13 +144,20 @@ func (m *Monitor) AggregateResult() types.NetworkCheckResult {
|
||||
defer m.mu.RUnlock()
|
||||
|
||||
result := types.NetworkCheckResult{
|
||||
LLMAPIReachable: true,
|
||||
DNSResolving: true,
|
||||
TCPReachable: true,
|
||||
LLMAPIReachable: true,
|
||||
EndpointsConfigured: len(m.status) > 0,
|
||||
DNSResolving: true,
|
||||
TCPReachable: true,
|
||||
}
|
||||
var totalLatency time.Duration
|
||||
checked := 0
|
||||
|
||||
if !result.EndpointsConfigured {
|
||||
// 无任何探活端点:不谎报"可达",标为未配置
|
||||
result.LLMAPIReachable = false
|
||||
result.Error = "no LLM endpoints configured for health check"
|
||||
}
|
||||
|
||||
for _, s := range m.status {
|
||||
if !s.Reachable {
|
||||
result.LLMAPIReachable = false
|
||||
|
||||
39
internal/network/monitor_test.go
Normal file
39
internal/network/monitor_test.go
Normal file
@ -0,0 +1,39 @@
|
||||
package network
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestAggregateResultNoEndpoints(t *testing.T) {
|
||||
m := NewMonitor(0)
|
||||
res := m.AggregateResult()
|
||||
if res.EndpointsConfigured {
|
||||
t.Error("no endpoints configured should be false")
|
||||
}
|
||||
if res.LLMAPIReachable {
|
||||
t.Error("LLMAPIReachable must not be true when no endpoints configured")
|
||||
}
|
||||
if res.Error == "" {
|
||||
t.Error("should report no-endpoints error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAggregateResultWithEndpointsUnreachable(t *testing.T) {
|
||||
m := NewMonitor(0)
|
||||
m.mu.Lock()
|
||||
m.status = []EndpointStatus{{URL: "http://127.0.0.1:1/", Reachable: false, Error: "dial refused"}}
|
||||
m.mu.Unlock()
|
||||
|
||||
res := m.AggregateResult()
|
||||
if !res.EndpointsConfigured {
|
||||
t.Error("endpoints configured should be true")
|
||||
}
|
||||
if res.LLMAPIReachable {
|
||||
t.Error("unreachable endpoint should make LLMAPIReachable false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllReachableEmpty(t *testing.T) {
|
||||
m := NewMonitor(0)
|
||||
if m.AllReachable() {
|
||||
t.Error("AllReachable must be false with no endpoints (not vacuously true)")
|
||||
}
|
||||
}
|
||||
@ -7,18 +7,19 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||
agentIO "gitcode.com/JianFeeeee/HomeAgent/internal/agent/io"
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/events"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/knowledge"
|
||||
luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory"
|
||||
doc "gitcode.com/JianFeeeee/HomeAgent/internal/memory/document"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/memory/text"
|
||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||
agentAPI "gitcode.com/JianFeeeee/HomeAgent/internal/agent/api"
|
||||
internalConfig "gitcode.com/JianFeeeee/HomeAgent/internal/config"
|
||||
luaVM "gitcode.com/JianFeeeee/HomeAgent/internal/lua"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||
)
|
||||
@ -69,17 +70,17 @@ type Registry struct {
|
||||
pluginAutoRestart map[string]bool
|
||||
sdkRefs map[string]*sdk.PluginSDK
|
||||
|
||||
iom *agentIO.IOManager
|
||||
evBus *events.Bus
|
||||
memDB *memory.GraphDB
|
||||
textMem *text.Memory
|
||||
docStore *doc.Store
|
||||
ks *knowledge.Store
|
||||
mgr *agentAPI.ProviderManager
|
||||
cfgReg *internalConfig.ConfigRegistry
|
||||
plgDir string
|
||||
lua *luaVM.VM
|
||||
baseKey string
|
||||
iom *agentIO.IOManager
|
||||
evBus *events.Bus
|
||||
memDB *memory.GraphDB
|
||||
textMem *text.Memory
|
||||
docStore *doc.Store
|
||||
ks *knowledge.Store
|
||||
mgr *agentAPI.ProviderManager
|
||||
cfgReg *internalConfig.ConfigRegistry
|
||||
plgDir string
|
||||
lua *luaVM.VM
|
||||
baseKey string
|
||||
|
||||
regTool sdk.ToolRegistrar
|
||||
regStage sdk.StageRegistrar
|
||||
@ -95,6 +96,7 @@ type Registry struct {
|
||||
idx *memory.Indexer
|
||||
|
||||
knownDisabled map[string]bool
|
||||
allowlist map[string]bool
|
||||
}
|
||||
|
||||
func NewRegistry() *Registry {
|
||||
@ -107,27 +109,51 @@ func NewRegistry() *Registry {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) SetIOManager(iom *agentIO.IOManager) { r.iom = iom }
|
||||
func (r *Registry) SetEventBus(evBus *events.Bus) { r.evBus = evBus }
|
||||
func (r *Registry) SetMemory(memDB *memory.GraphDB) { r.memDB = memDB }
|
||||
func (r *Registry) SetTextMemory(tm *text.Memory) { r.textMem = tm }
|
||||
func (r *Registry) SetDocStore(ds *doc.Store) { r.docStore = ds }
|
||||
func (r *Registry) SetKnowledge(ks *knowledge.Store) { r.ks = ks }
|
||||
func (r *Registry) SetProviderManager(mgr *agentAPI.ProviderManager) { r.mgr = mgr }
|
||||
func (r *Registry) SetIOManager(iom *agentIO.IOManager) { r.iom = iom }
|
||||
func (r *Registry) SetEventBus(evBus *events.Bus) { r.evBus = evBus }
|
||||
func (r *Registry) SetMemory(memDB *memory.GraphDB) { r.memDB = memDB }
|
||||
func (r *Registry) SetTextMemory(tm *text.Memory) { r.textMem = tm }
|
||||
func (r *Registry) SetDocStore(ds *doc.Store) { r.docStore = ds }
|
||||
func (r *Registry) SetKnowledge(ks *knowledge.Store) { r.ks = ks }
|
||||
func (r *Registry) SetProviderManager(mgr *agentAPI.ProviderManager) { r.mgr = mgr }
|
||||
func (r *Registry) SetConfigRegistry(cfgReg *internalConfig.ConfigRegistry) { r.cfgReg = cfgReg }
|
||||
func (r *Registry) SetPluginDir(dir string) { r.plgDir = dir }
|
||||
func (r *Registry) SetLuaVM(vm *luaVM.VM) { r.lua = vm }
|
||||
func (r *Registry) SetBaseAPIKey(key string) { r.baseKey = key }
|
||||
func (r *Registry) SetToolRegistrar(fn sdk.ToolRegistrar) { r.regTool = fn }
|
||||
func (r *Registry) SetStageRegistrar(fn sdk.StageRegistrar) { r.regStage = fn }
|
||||
func (r *Registry) SetAPIRegistrar(fn sdk.APIRegistrar) { r.regAPI = fn }
|
||||
func (r *Registry) SetToolCleaner(tc PluginToolCleaner) { r.toolCleaner = tc }
|
||||
func (r *Registry) SetStatusProvider(sp sdk.StatusAPI) { r.status = sp }
|
||||
func (r *Registry) SetSupervisor(sup sdk.SupervisorAPI) { r.sup = sup }
|
||||
func (r *Registry) SetTracker(trk *tracker.Tracker) { r.trk = trk }
|
||||
func (r *Registry) SetConfig(cfg *types.Config) { r.cfg = cfg }
|
||||
func (r *Registry) SetStageHost(sh sdk.ToolSource) { r.stageHost = sh }
|
||||
func (r *Registry) SetIndexer(idx *memory.Indexer) { r.idx = idx }
|
||||
func (r *Registry) SetPluginDir(dir string) { r.plgDir = dir }
|
||||
func (r *Registry) SetLuaVM(vm *luaVM.VM) { r.lua = vm }
|
||||
func (r *Registry) SetBaseAPIKey(key string) { r.baseKey = key }
|
||||
func (r *Registry) SetToolRegistrar(fn sdk.ToolRegistrar) { r.regTool = fn }
|
||||
func (r *Registry) SetStageRegistrar(fn sdk.StageRegistrar) { r.regStage = fn }
|
||||
func (r *Registry) SetAPIRegistrar(fn sdk.APIRegistrar) { r.regAPI = fn }
|
||||
func (r *Registry) SetToolCleaner(tc PluginToolCleaner) { r.toolCleaner = tc }
|
||||
func (r *Registry) SetStatusProvider(sp sdk.StatusAPI) { r.status = sp }
|
||||
func (r *Registry) SetSupervisor(sup sdk.SupervisorAPI) { r.sup = sup }
|
||||
func (r *Registry) SetTracker(trk *tracker.Tracker) { r.trk = trk }
|
||||
func (r *Registry) SetConfig(cfg *types.Config) { r.cfg = cfg }
|
||||
func (r *Registry) SetStageHost(sh sdk.ToolSource) { r.stageHost = sh }
|
||||
func (r *Registry) SetIndexer(idx *memory.Indexer) { r.idx = idx }
|
||||
|
||||
// SetLoadAllowlist 限制 Load 仅装载指定插件名(failback 受限启动用)。
|
||||
// 空/未设置 = 装载全部。违反白名单的插件(含已注册工厂)一律跳过。
|
||||
func (r *Registry) SetLoadAllowlist(names []string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.allowlist = nil
|
||||
if len(names) == 0 {
|
||||
return
|
||||
}
|
||||
r.allowlist = make(map[string]bool, len(names))
|
||||
for _, n := range names {
|
||||
r.allowlist[strings.TrimSpace(n)] = true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) allowlistAllows(name string) bool {
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if r.allowlist == nil {
|
||||
return true
|
||||
}
|
||||
return r.allowlist[name]
|
||||
}
|
||||
|
||||
func (r *Registry) RegisterNative(name string, factory NativeFactory) {
|
||||
r.mu.Lock()
|
||||
@ -144,13 +170,13 @@ type channelDevice struct {
|
||||
chDef agentIO.ChannelDef
|
||||
}
|
||||
|
||||
func (d *channelDevice) Name() string { return d.name }
|
||||
func (d *channelDevice) Type() agentIO.DeviceType { return agentIO.DeviceOutput }
|
||||
func (d *channelDevice) Description() string { return d.desc }
|
||||
func (d *channelDevice) Start() error { return nil }
|
||||
func (d *channelDevice) Stop() error { return nil }
|
||||
func (d *channelDevice) Name() string { return d.name }
|
||||
func (d *channelDevice) Type() agentIO.DeviceType { return agentIO.DeviceOutput }
|
||||
func (d *channelDevice) Description() string { return d.desc }
|
||||
func (d *channelDevice) Start() error { return nil }
|
||||
func (d *channelDevice) Stop() error { return nil }
|
||||
func (d *channelDevice) OutputCapabilities() agentIO.OutputCapability { return d.caps }
|
||||
func (d *channelDevice) Tools() []agentIO.ToolDef { return nil }
|
||||
func (d *channelDevice) Tools() []agentIO.ToolDef { return nil }
|
||||
func (d *channelDevice) Execute(tool string, args map[string]interface{}) (interface{}, error) {
|
||||
return d.handler(args)
|
||||
}
|
||||
@ -243,6 +269,9 @@ func (r *Registry) Load(dir string) error {
|
||||
continue
|
||||
}
|
||||
name := entry.Name()
|
||||
if !r.allowlistAllows(name) {
|
||||
continue
|
||||
}
|
||||
plgDir := filepath.Join(dir, name)
|
||||
if r.loadOne(plgDir, name) {
|
||||
loaded[name] = true
|
||||
@ -269,6 +298,9 @@ func (r *Registry) Load(dir string) error {
|
||||
if loaded[name] {
|
||||
continue
|
||||
}
|
||||
if !r.allowlistAllows(name) {
|
||||
continue
|
||||
}
|
||||
if r.isDisabled(name) {
|
||||
log.Printf("[plugin] %s is disabled, skipping", name)
|
||||
r.mu.Lock()
|
||||
|
||||
@ -12,6 +12,7 @@ import (
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/plugin"
|
||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/system"
|
||||
)
|
||||
|
||||
func init() {
|
||||
@ -26,6 +27,7 @@ type Plugin struct {
|
||||
sdk *sdk.PluginSDK
|
||||
mu sync.RWMutex
|
||||
filesDir string
|
||||
baseDir string // L0 写前留档根目录(<data>/file_baseline 的父目录),空则禁用
|
||||
}
|
||||
|
||||
func New(name string) *Plugin {
|
||||
@ -57,6 +59,29 @@ func (p *Plugin) Start(s *sdk.PluginSDK) error {
|
||||
}
|
||||
p.filesDir = abs
|
||||
|
||||
// L0 写前留档:主 agent 写 /etc 等受保护路径前自动存档原文(homed 注入 <data>)
|
||||
if cfg := s.Config().Get(); cfg != nil {
|
||||
if dd := cfg.Daemon.DataDir; dd != "" {
|
||||
p.baseDir = dd
|
||||
}
|
||||
}
|
||||
s.Settings().RegisterDef(sdk.ConfigDef{
|
||||
Key: "baseline_dir",
|
||||
Default: p.baseDir,
|
||||
Type: "string",
|
||||
DisplayName: "写前留档目录",
|
||||
Description: "写受保护系统路径前自动存档原文的目录(空禁用)",
|
||||
Category: "files",
|
||||
})
|
||||
if v, err := s.Settings().Get("baseline_dir"); err == nil && v != nil {
|
||||
if s, ok := v.(string); ok && s != "" {
|
||||
p.baseDir = s
|
||||
}
|
||||
}
|
||||
if p.baseDir != "" {
|
||||
log.Printf("[%s] write-ahead baseline dir: %s", p.name, system.FileBaselineDir(p.baseDir))
|
||||
}
|
||||
|
||||
tp := p.name + "_"
|
||||
|
||||
s.RegisterTool(tp+"read", sdk.ToolDef{
|
||||
@ -251,6 +276,15 @@ func (p *Plugin) handleWrite(args map[string]interface{}) (interface{}, error) {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
|
||||
// L0 写前留档:覆盖已有受保护文件前,原文存档供 guard 还原。
|
||||
if mode != "create" {
|
||||
if archived, aerr := p.archiveBeforeWrite(absPath); aerr != nil {
|
||||
log.Printf("[%s] write-ahead archive %s: %v", p.name, absPath, aerr)
|
||||
} else if archived {
|
||||
log.Printf("[%s] write-ahead archived %s", p.name, absPath)
|
||||
}
|
||||
}
|
||||
|
||||
switch mode {
|
||||
case "create":
|
||||
if _, err := os.Stat(absPath); err == nil {
|
||||
@ -337,6 +371,13 @@ func (p *Plugin) handleEdit(args map[string]interface{}) (interface{}, error) {
|
||||
return errorResult(err.Error()), nil
|
||||
}
|
||||
|
||||
// L0 写前留档(edit 可能覆盖已存在文件)
|
||||
if archived, aerr := p.archiveBeforeWrite(absPath); aerr != nil {
|
||||
log.Printf("[%s] write-ahead archive %s: %v", p.name, absPath, aerr)
|
||||
} else if archived {
|
||||
log.Printf("[%s] write-ahead archived %s", p.name, absPath)
|
||||
}
|
||||
|
||||
rawEdits, ok := args["edits"].([]interface{})
|
||||
if !ok || len(rawEdits) == 0 {
|
||||
return errorResult("edits must be a non-empty array"), nil
|
||||
@ -481,6 +522,14 @@ func errorResult(msg string) map[string]interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
// archiveBeforeWrite L0 写前留档:若目标为受保护系统路径且已存在,则存档原文。
|
||||
func (p *Plugin) archiveBeforeWrite(absPath string) (bool, error) {
|
||||
if p.baseDir == "" {
|
||||
return false, nil
|
||||
}
|
||||
return system.ArchiveBeforeWrite(p.baseDir, absPath)
|
||||
}
|
||||
|
||||
func getSetting[T any](s sdk.SettingsAPI, key string, def T) T {
|
||||
v, err := s.Get(key)
|
||||
if err != nil || v == nil {
|
||||
|
||||
233
internal/recovery/recovery.go
Normal file
233
internal/recovery/recovery.go
Normal file
@ -0,0 +1,233 @@
|
||||
// Package recovery 实现 L1 failback 的确定性恢复梯子与 guard↔worker 之间的
|
||||
// 恢复任务/结果文件协议。整个梯子无 token 消耗(除 rescue 源 QuickChat 复检外):
|
||||
// probe(main 可达) → 还原 DNS/proxy → probe → 还原 config 快照+ReloadFromConfig → probe
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Brightness 恢复梯子各阶段的判定强度(越低越优先主张)。
|
||||
type Brightness int
|
||||
|
||||
const (
|
||||
BrightDeterministic Brightness = iota // 无推理
|
||||
BrightMinimal // 最小推理(未知/混合分支)
|
||||
)
|
||||
|
||||
// RescueSource guard.yaml 中配置的救援 LLM 源(锚定 IP,绕开被破坏的 DNS/代理)。
|
||||
type RescueSource struct {
|
||||
Name string `yaml:"name" json:"name"`
|
||||
BaseURL string `yaml:"base_url" json:"base_url"`
|
||||
APIKey string `yaml:"api_key" json:"api_key"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Adapter string `yaml:"adapter" json:"adapter"`
|
||||
Thinking bool `yaml:"thinking_enabled" json:"thinking_enabled"`
|
||||
}
|
||||
|
||||
// Task guard 写、failback worker 读的恢复任务(跨进程文件协议)。
|
||||
type Task struct {
|
||||
Attempt int `json:"attempt"`
|
||||
MaxAttempts int `json:"max_attempts"`
|
||||
AttemptTimeout string `json:"attempt_timeout"` // 单轮超时,如 "120s"
|
||||
TriggerPrompt string `json:"trigger_prompt,omitempty"`
|
||||
KnowledgeBase string `json:"knowledge_base,omitempty"`
|
||||
RescueSource RescueSource `json:"rescue_source"`
|
||||
PluginList []string `json:"plugins,omitempty"` // failback 插件集(缺省回退 config.db)
|
||||
}
|
||||
|
||||
// Plugins 返回 failback 插件集;为空则返回 nil。
|
||||
func (t *Task) Plugins() []string { return t.PluginList }
|
||||
|
||||
// TaskPath 返回 <dataDir>/recovery/recovery_task.json。
|
||||
func TaskPath(dataDir string) string {
|
||||
return filepath.Join(dataDir, "recovery", "recovery_task.json")
|
||||
}
|
||||
|
||||
// SaveTask 持久化恢复任务。
|
||||
func SaveTask(path string, t *Task) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(t, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
// LoadTask 读回恢复任务。
|
||||
func LoadTask(path string) (*Task, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var t Task
|
||||
if err := json.Unmarshal(data, &t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// Result recovery worker 单次 failback 尝试的结论,写给 guard 读取。
|
||||
type Result struct {
|
||||
Attempt int `json:"attempt"`
|
||||
Success bool `json:"success"`
|
||||
Cause string `json:"cause"` // 判定出的根因类(取 diag / 梯子命中)
|
||||
NetworkOK bool `json:"network_ok"` // 网络(resolv/hosts)是否与基线一致
|
||||
RestoredNet []string `json:"restored_net"` // 已还原的网络文件
|
||||
ConfigRestored bool `json:"config_restored"` // 是否还原 LLM 配置快照
|
||||
QuickChatOK bool `json:"quickchat_ok"` // 最后用 rescue 源复检是否可达
|
||||
Steps []string `json:"steps"` // 走过的恢复步骤
|
||||
Message string `json:"message"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
}
|
||||
|
||||
// Quote 生成一行可读摘要(写入 worker 日志 / guard 决策用)。
|
||||
func (r *Result) Quote() string {
|
||||
return fmt.Sprintf("success=%v cause=%q net_ok=%v config_restored=%v quickchat=%v steps=%d",
|
||||
r.Success, r.Cause, r.NetworkOK, r.ConfigRestored, r.QuickChatOK, len(r.Steps))
|
||||
}
|
||||
|
||||
// Prober 探测 LLM 源是否可达(QuickChat)。用于梯子各阶段复检。
|
||||
type Prober func(ctx context.Context, label string) (bool, error)
|
||||
|
||||
// NetworkRestorer 还原 DNS/proxy。返回改动列表。
|
||||
type NetworkRestorer func() ([]string, error)
|
||||
|
||||
// ConfigRestorer 还原 LLM 配置快照并 ReloadFromConfig。返回精度(改动键数)。
|
||||
type ConfigRestorer func() (int, error)
|
||||
|
||||
// Ladder 确定性恢复梯子。
|
||||
type Ladder struct {
|
||||
ResultFile string
|
||||
Attempt int
|
||||
Probe Prober
|
||||
RestoreNetwork NetworkRestorer
|
||||
RestoreConfig ConfigRestorer
|
||||
Log func(format string, args ...interface{})
|
||||
}
|
||||
|
||||
// Run 执行一次恢复梯子并返回结论(同时持久化到 ResultFile)。错误仅表示梯子
|
||||
// 自身失败;结论成败由 Result.Success 表达。
|
||||
func (l *Ladder) Run(ctx context.Context) *Result {
|
||||
res := &Result{
|
||||
Attempt: l.Attempt,
|
||||
Cause: "unknown",
|
||||
Timestamp: time.Now(),
|
||||
}
|
||||
l.addStep(res, "begin attempt")
|
||||
|
||||
// 1. 初始探测:quickchat 已通 → 无需恢复(网络与配置至少一个坏,但 rescue 可达)
|
||||
ok, err := l.Probe(ctx, "initial")
|
||||
if err == nil && ok {
|
||||
res.Success = true
|
||||
res.Cause = "healthy"
|
||||
l.addStep(res, "initial probe: LLM reachable")
|
||||
l.finish(res)
|
||||
return res
|
||||
}
|
||||
res.Cause = "unreachable"
|
||||
l.addStep(res, "initial probe: unreachable: %v", err)
|
||||
|
||||
// 2. 还原 DNS/proxy(L1 第一步小修命中即停)
|
||||
if nested, rerr := l.RestoreNetwork(); rerr != nil {
|
||||
l.addStep(res, "network restore error: %v", rerr)
|
||||
} else {
|
||||
res.RestoredNet = nested
|
||||
res.NetworkOK = len(nested) == 0
|
||||
l.addStep(res, "network restore: %d changed", len(nested))
|
||||
}
|
||||
if ok, _ = l.Probe(ctx, "after-network"); err == nil && ok {
|
||||
res.Success = true
|
||||
res.Cause = "network_config"
|
||||
l.addStep(res, "post-network probe: reachable")
|
||||
l.finish(res)
|
||||
return res
|
||||
}
|
||||
l.addStep(res, "post-network probe: unreachable")
|
||||
|
||||
// 3. 还原 config 快照 + ReloadFromConfig(L1 第二步)
|
||||
if n, cerr := l.RestoreConfig(); cerr != nil {
|
||||
l.addStep(res, "config restore error: %v", cerr)
|
||||
} else {
|
||||
res.ConfigRestored = n > 0
|
||||
l.addStep(res, "config restore: %d keys", n)
|
||||
}
|
||||
if ok, _ = l.Probe(ctx, "after-config"); err == nil && ok {
|
||||
res.Success = true
|
||||
res.Cause = "config"
|
||||
l.addStep(res, "post-config probe: reachable")
|
||||
l.finish(res)
|
||||
return res
|
||||
}
|
||||
l.addStep(res, "post-config probe: unreachable")
|
||||
res.QuickChatOK = false
|
||||
l.addStep(res, "ladder exhausted")
|
||||
|
||||
l.finish(res)
|
||||
return res
|
||||
}
|
||||
|
||||
func (l *Ladder) addStep(res *Result, format string, args ...interface{}) {
|
||||
msg := fmt.Sprintf(format, args...)
|
||||
res.Steps = append(res.Steps, msg)
|
||||
if l.Log != nil {
|
||||
l.Log("[recovery] %s", msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Ladder) finish(res *Result) {
|
||||
res.QuickChatOK = res.Success
|
||||
if l.ResultFile != "" {
|
||||
if err := SaveResult(l.ResultFile, res); err != nil {
|
||||
if l.Log != nil {
|
||||
l.Log("[recovery] save result: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SaveResult 把恢复结论持久化到文件(guard 读取)。
|
||||
func SaveResult(path string, r *Result) error {
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(r, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
// LoadResult 读回恢复结论。
|
||||
func LoadResult(path string) (*Result, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var r Result
|
||||
if err := json.Unmarshal(data, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
// ResultPath 返回 <dataDir>/recovery/recovery_result.json。
|
||||
func ResultPath(dataDir string) string {
|
||||
return filepath.Join(dataDir, "recovery", "recovery_result.json")
|
||||
}
|
||||
|
||||
// SortSteps 供测试/展示封装(保序打印)。
|
||||
func SortSteps(s []string) []string {
|
||||
out := make([]string, len(s))
|
||||
copy(out, s)
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
208
internal/recovery/recovery_test.go
Normal file
208
internal/recovery/recovery_test.go
Normal file
@ -0,0 +1,208 @@
|
||||
package recovery
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeProber struct {
|
||||
results map[string]bool // label → ok
|
||||
mu chan struct{}
|
||||
}
|
||||
|
||||
func newFakeProber() *fakeProber { return &fakeProber{results: map[string]bool{}} }
|
||||
|
||||
func (f *fakeProber) set(label string, ok bool) {
|
||||
if f.results == nil {
|
||||
f.results = map[string]bool{}
|
||||
}
|
||||
f.results[label] = ok
|
||||
}
|
||||
|
||||
func (f *fakeProber) probe(ctx context.Context, label string) (bool, error) {
|
||||
if f.results == nil {
|
||||
return false, nil
|
||||
}
|
||||
ok, _ := f.results[label]
|
||||
return ok, nil
|
||||
}
|
||||
|
||||
func TestLadderHealthy(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := newFakeProber()
|
||||
p.set("initial", true) // 初始即通,不触发任何恢复
|
||||
l := &Ladder{
|
||||
ResultFile: ResultPath(dir),
|
||||
Attempt: 1,
|
||||
Probe: p.probe,
|
||||
RestoreNetwork: func() ([]string, error) {
|
||||
t.Fatal("network restore should not be called when healthy")
|
||||
return nil, nil
|
||||
},
|
||||
RestoreConfig: func() (int, error) {
|
||||
t.Fatal("config restore should not be called when healthy")
|
||||
return 0, nil
|
||||
},
|
||||
}
|
||||
res := l.Run(context.Background())
|
||||
if !res.Success {
|
||||
t.Fatalf("expected success, got %s", res.Quote())
|
||||
}
|
||||
if res.Cause != "healthy" {
|
||||
t.Fatalf("cause = %q, want healthy", res.Cause)
|
||||
}
|
||||
if len(res.Steps) != 2 {
|
||||
t.Fatalf("steps = %d, want 2", len(res.Steps))
|
||||
}
|
||||
// 结果落盘
|
||||
onDisk, err := LoadResult(ResultPath(dir))
|
||||
if err != nil {
|
||||
t.Fatalf("LoadResult: %v", err)
|
||||
}
|
||||
if !onDisk.Success {
|
||||
t.Fatal("disk result not success")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLadderNetworkRecovery(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := newFakeProber()
|
||||
p.set("initial", false)
|
||||
p.set("after-network", true) // 还原网络后通了
|
||||
var netCalls int
|
||||
l := &Ladder{
|
||||
ResultFile: ResultPath(dir),
|
||||
Attempt: 2,
|
||||
Probe: p.probe,
|
||||
RestoreNetwork: func() ([]string, error) {
|
||||
netCalls++
|
||||
return []string{"/etc/resolv.conf"}, nil
|
||||
},
|
||||
RestoreConfig: func() (int, error) {
|
||||
t.Fatal("config restore should not run when network fixes it")
|
||||
return 0, nil
|
||||
},
|
||||
}
|
||||
res := l.Run(context.Background())
|
||||
if !res.Success {
|
||||
t.Fatalf("expected success, got %s", res.Quote())
|
||||
}
|
||||
if res.Cause != "network_config" {
|
||||
t.Fatalf("cause = %q, want network_config", res.Cause)
|
||||
}
|
||||
if netCalls != 1 {
|
||||
t.Fatalf("network restore calls = %d, want 1", netCalls)
|
||||
}
|
||||
if len(res.RestoredNet) != 1 {
|
||||
t.Fatalf("RestoredNet = %v, want 1 entry", res.RestoredNet)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLadderConfigRecovery(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := newFakeProber()
|
||||
p.set("initial", false)
|
||||
p.set("after-network", false)
|
||||
p.set("after-config", true)
|
||||
var cfgCalls int
|
||||
var netCalls int
|
||||
l := &Ladder{
|
||||
ResultFile: ResultPath(dir),
|
||||
Attempt: 3,
|
||||
Probe: p.probe,
|
||||
RestoreNetwork: func() ([]string, error) {
|
||||
netCalls++
|
||||
return nil, nil
|
||||
},
|
||||
RestoreConfig: func() (int, error) {
|
||||
cfgCalls++
|
||||
return 7, nil
|
||||
},
|
||||
}
|
||||
res := l.Run(context.Background())
|
||||
if !res.Success {
|
||||
t.Fatalf("expected success, got %s", res.Quote())
|
||||
}
|
||||
if res.Cause != "config" {
|
||||
t.Fatalf("cause = %q, want config", res.Cause)
|
||||
}
|
||||
if cfgCalls != 1 || netCalls != 1 {
|
||||
t.Fatalf("cfg=%d net=%d, want both 1", cfgCalls, netCalls)
|
||||
}
|
||||
if !res.ConfigRestored {
|
||||
t.Fatal("ConfigRestored should be true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLadderExhausted(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
p := newFakeProber()
|
||||
p.set("initial", false)
|
||||
p.set("after-network", false)
|
||||
p.set("after-config", false)
|
||||
l := &Ladder{
|
||||
ResultFile: ResultPath(dir),
|
||||
Attempt: 1,
|
||||
Probe: p.probe,
|
||||
RestoreNetwork: func() ([]string, error) { return nil, nil },
|
||||
RestoreConfig: func() (int, error) { return 0, nil },
|
||||
}
|
||||
res := l.Run(context.Background())
|
||||
if res.Success {
|
||||
t.Fatal("expected failure when ladder exhausted")
|
||||
}
|
||||
if res.Cause != "unreachable" {
|
||||
t.Fatalf("cause = %q, want unreachable", res.Cause)
|
||||
}
|
||||
if res.QuickChatOK {
|
||||
t.Fatal("QuickChatOK should be false on failure")
|
||||
}
|
||||
if len(res.Steps) < 6 {
|
||||
t.Fatalf("steps = %d, want >= 6", len(res.Steps))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTaskFileRoundtrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := TaskPath(dir)
|
||||
task := &Task{
|
||||
Attempt: 2,
|
||||
MaxAttempts: 4,
|
||||
AttemptTimeout: "120s",
|
||||
TriggerPrompt: "你是恢复 agent",
|
||||
RescueSource: RescueSource{Name: "rescue", BaseURL: "http://1.2.3.4:8080", Adapter: "openai"},
|
||||
PluginList: []string{"webui", "recoverydiag"},
|
||||
}
|
||||
if err := SaveTask(path, task); err != nil {
|
||||
t.Fatalf("SaveTask: %v", err)
|
||||
}
|
||||
loaded, err := LoadTask(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadTask: %v", err)
|
||||
}
|
||||
if loaded.RescueSource.BaseURL != task.RescueSource.BaseURL {
|
||||
t.Fatal("rescue base_url mismatch")
|
||||
}
|
||||
if len(loaded.Plugins()) != 2 {
|
||||
t.Fatalf("plugins = %v, want 2", loaded.Plugins())
|
||||
}
|
||||
_ = os.Remove(path)
|
||||
}
|
||||
|
||||
func TestResultFileRoundtrip(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "recovery", "result.json")
|
||||
r := &Result{Attempt: 1, Success: true, Cause: "network_config", QuickChatOK: true}
|
||||
if err := SaveResult(path, r); err != nil {
|
||||
t.Fatalf("SaveResult: %v", err)
|
||||
}
|
||||
loaded, err := LoadResult(path)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadResult: %v", err)
|
||||
}
|
||||
if !loaded.Success || loaded.Cause != "network_config" {
|
||||
t.Fatalf("roundtrip mismatch: %+v", loaded)
|
||||
}
|
||||
}
|
||||
@ -8,21 +8,23 @@ import (
|
||||
"time"
|
||||
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/network"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
sdk "gitcode.com/JianFeeeee/HomeAgent/internal/sdk"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/internal/tracker"
|
||||
"gitcode.com/JianFeeeee/HomeAgent/pkg/types"
|
||||
)
|
||||
|
||||
const directAgentID types.AgentID = "main"
|
||||
|
||||
type Daemon struct {
|
||||
cfg *types.Config
|
||||
nm *network.Monitor
|
||||
trk *tracker.Tracker
|
||||
agents map[types.AgentID]*agentInstance
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
cfg *types.Config
|
||||
nm *network.Monitor
|
||||
trk *tracker.Tracker
|
||||
agents map[types.AgentID]*agentInstance
|
||||
mu sync.RWMutex
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
heartbeatSource func(id types.AgentID) (time.Time, types.HealthStatus, error)
|
||||
restartHandler func(id types.AgentID)
|
||||
}
|
||||
|
||||
type agentInstance struct {
|
||||
@ -51,6 +53,21 @@ func (d *Daemon) SetTracker(trk *tracker.Tracker) {
|
||||
d.trk = trk
|
||||
}
|
||||
|
||||
// SetHeartbeatSource 接入 agent 真实存活探测源(direct 模式下为同进程 agent core 状态)。
|
||||
// 成功返回时 lastHB 才更新;无源时 agent 健康保持未知。
|
||||
func (d *Daemon) SetHeartbeatSource(fn func(id types.AgentID) (time.Time, types.HealthStatus, error)) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.heartbeatSource = fn
|
||||
}
|
||||
|
||||
// SetRestartHandler 接入真实重启动作(direct 模式下由 homed 注册"清理后以特殊码退出",交给 guard/systemd 重建)。
|
||||
func (d *Daemon) SetRestartHandler(fn func(id types.AgentID)) {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
d.restartHandler = fn
|
||||
}
|
||||
|
||||
func (d *Daemon) Start() error {
|
||||
log.Println("[homed] starting HomeAgent daemon")
|
||||
|
||||
@ -127,18 +144,35 @@ func (d *Daemon) checkAllAgents() {
|
||||
}
|
||||
|
||||
func (d *Daemon) checkAgent(id types.AgentID, agent *agentInstance) {
|
||||
netStatus := d.nm.AggregateResult()
|
||||
// 1) agent 存活源:只有确认存活才更新 lastHB
|
||||
d.mu.RLock()
|
||||
hbSrc := d.heartbeatSource
|
||||
d.mu.RUnlock()
|
||||
|
||||
if !netStatus.LLMAPIReachable {
|
||||
agent.health = types.HealthDegraded
|
||||
agent.failCount++
|
||||
log.Printf("[homed] agent %s: LLM API unreachable (fail %d)", id, agent.failCount)
|
||||
if hbSrc != nil {
|
||||
hbTime, hbHealth, err := hbSrc(id)
|
||||
if err != nil || hbTime.IsZero() {
|
||||
agent.health = types.HealthDown
|
||||
log.Printf("[homed] agent %s heartbeat lost: %v", id, err)
|
||||
} else {
|
||||
agent.lastHB = hbTime
|
||||
agent.health = hbHealth
|
||||
}
|
||||
} else {
|
||||
agent.health = types.HealthHealthy
|
||||
agent.failCount = 0
|
||||
agent.health = types.HealthUnknown
|
||||
}
|
||||
|
||||
agent.lastHB = time.Now()
|
||||
// 2) 网络:仅在配置了探活端点时才据此判定降级
|
||||
netStatus := d.nm.AggregateResult()
|
||||
if netStatus.EndpointsConfigured && !netStatus.LLMAPIReachable {
|
||||
if agent.health != types.HealthDown {
|
||||
agent.health = types.HealthDegraded
|
||||
}
|
||||
agent.failCount++
|
||||
log.Printf("[homed] agent %s: LLM API unreachable (fail %d)", id, agent.failCount)
|
||||
} else if agent.health == types.HealthHealthy {
|
||||
agent.failCount = 0
|
||||
}
|
||||
|
||||
if agent.failCount >= agent.cfg.RollbackPolicy.MaxRetries {
|
||||
d.handleFailure(id, agent)
|
||||
@ -164,13 +198,23 @@ func (d *Daemon) handleFailure(id types.AgentID, agent *agentInstance) {
|
||||
}
|
||||
|
||||
func (d *Daemon) restartAgent(id types.AgentID, agent *agentInstance) {
|
||||
log.Printf("[homed] resetting agent %s", id)
|
||||
|
||||
agent.state = types.AgentStateStopped
|
||||
d.RegisterAgent(id)
|
||||
|
||||
agent.failCount = 0
|
||||
log.Printf("[homed] agent %s reset", id)
|
||||
|
||||
d.mu.RLock()
|
||||
handler := d.restartHandler
|
||||
d.mu.RUnlock()
|
||||
|
||||
if handler != nil {
|
||||
log.Printf("[homed] agent %s restart handler invoked", id)
|
||||
handler(id)
|
||||
return
|
||||
}
|
||||
|
||||
// 无真实重启通道:退回内存态复位(记录,不再假装成功)
|
||||
d.RegisterAgent(id)
|
||||
agent.failCount = 0
|
||||
log.Printf("[homed] agent %s reset in-memory only (no restart handler registered)", id)
|
||||
}
|
||||
|
||||
func (d *Daemon) GetAgentStatus(id types.AgentID) (*AgentStatus, error) {
|
||||
|
||||
@ -184,7 +184,7 @@ func TestRegisterAgentMultiple(t *testing.T) {
|
||||
d := New(cfg)
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
d.RegisterAgent(types.AgentID(string(rune('a'+i))))
|
||||
d.RegisterAgent(types.AgentID(string(rune('a' + i))))
|
||||
}
|
||||
|
||||
agents := d.ListAgents()
|
||||
@ -207,3 +207,64 @@ func TestConcurrentAccess(t *testing.T) {
|
||||
go d.RegisterAgent(types.AgentID(string(rune('a' + i))))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckAgentHeartbeatSource(t *testing.T) {
|
||||
cfg := &types.Config{
|
||||
Daemon: types.DaemonConfig{CheckInterval: time.Minute, HeartbeatInterval: 30 * time.Second},
|
||||
}
|
||||
cfg.Defaults.RollbackPolicy.MaxRetries = 3
|
||||
d := New(cfg)
|
||||
d.RegisterAgent("main")
|
||||
a := d.agents["main"]
|
||||
|
||||
d.SetHeartbeatSource(func(id types.AgentID) (time.Time, types.HealthStatus, error) {
|
||||
return time.Now(), types.HealthHealthy, nil
|
||||
})
|
||||
|
||||
regHB := a.lastHB
|
||||
if regHB.IsZero() {
|
||||
t.Fatal("lastHB should be set at register")
|
||||
}
|
||||
|
||||
d.checkAgent("main", a)
|
||||
if a.lastHB.Before(regHB) {
|
||||
t.Fatal("lastHB should update after a successful heartbeat poll")
|
||||
}
|
||||
if a.health != types.HealthHealthy {
|
||||
t.Fatalf("expected healthy, got %v", a.health)
|
||||
}
|
||||
|
||||
// 健康源丢失:lastHB 不应再更新,状态置 Down
|
||||
lastHB := a.lastHB
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
d.SetHeartbeatSource(func(id types.AgentID) (time.Time, types.HealthStatus, error) {
|
||||
return time.Time{}, types.HealthDown, nil
|
||||
})
|
||||
d.checkAgent("main", a)
|
||||
if a.health != types.HealthDown {
|
||||
t.Fatalf("expected down, got %v", a.health)
|
||||
}
|
||||
if !a.lastHB.Equal(lastHB) {
|
||||
t.Fatal("lastHB must NOT update when the agent does not respond")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestartHandlerInvoked(t *testing.T) {
|
||||
cfg := &types.Config{
|
||||
Daemon: types.DaemonConfig{CheckInterval: time.Minute, HeartbeatInterval: 30 * time.Second},
|
||||
}
|
||||
cfg.Defaults.RollbackPolicy.MaxRetries = 3
|
||||
d := New(cfg)
|
||||
d.RegisterAgent("main")
|
||||
a := d.agents["main"]
|
||||
|
||||
called := false
|
||||
d.SetRestartHandler(func(id types.AgentID) { called = true })
|
||||
d.restartAgent("main", a)
|
||||
if !called {
|
||||
t.Fatal("restart handler should be invoked")
|
||||
}
|
||||
if a.failCount != 0 {
|
||||
t.Fatalf("failCount should reset, got %d", a.failCount)
|
||||
}
|
||||
}
|
||||
|
||||
146
internal/system/guard.go
Normal file
146
internal/system/guard.go
Normal file
@ -0,0 +1,146 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ProtectedPaths L0 写前留档保护的系统路径前缀。主 agent 通过 files 插件写
|
||||
// 这些路径前,先自动把原文存档到 <dataDir>/file_baseline/,供 guard 还原。
|
||||
//
|
||||
// 该集合强依赖当前 Linux 发行版布局(/etc、/home)。默认值仅作兜底;
|
||||
// 生产环境应由 homed/main 或 guard.yaml 显式注入(SetProtectedPaths),
|
||||
// 以适配具体发行版/部署路径,避免硬编码失效。
|
||||
var protectedPrefixes = []string{
|
||||
"/etc/",
|
||||
}
|
||||
|
||||
// SetProtectedPaths 覆盖受保护路径前缀集合(发行版/部署路径适配入口)。
|
||||
// 幂等地保留 / 前缀兜底;传入 nil/空则恢复默认。
|
||||
func SetProtectedPaths(prefixes []string) {
|
||||
if len(prefixes) == 0 {
|
||||
protectedPrefixes = []string{"/etc/"}
|
||||
return
|
||||
}
|
||||
ns := make([]string, 0, len(prefixes))
|
||||
for _, p := range prefixes {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
ns = append(ns, p)
|
||||
}
|
||||
}
|
||||
protectedPrefixes = ns
|
||||
}
|
||||
|
||||
// ProtectedPaths 返回当前受保护路径前缀(副本)。
|
||||
func ProtectedPaths() []string {
|
||||
out := make([]string, len(protectedPrefixes))
|
||||
copy(out, protectedPrefixes)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsProtectedPath 判断路径是否属于受保护的写前留档范围。
|
||||
func IsProtectedPath(path string) bool {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, p := range protectedPrefixes {
|
||||
if strings.HasPrefix(abs, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsProtectedPathExplicit 同 IsProtectedPath,但仅匹配传入的显式前缀集
|
||||
// (不改全局状态时的判断入口,供测试/工具使用)。
|
||||
func IsProtectedPathExplicit(prefixes []string, path string) bool {
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(abs, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ArchiveBeforeWrite 在覆盖某个受保护路径前调用:若目标为已存在的普通文件,
|
||||
// 则把原文复制到 <dataDir>/file_baseline/<相对路径>(幂等:已存在同 hash 则跳过),
|
||||
// 返回是否发生了留档。用于 L0 写 /etc 前的自动留档。
|
||||
//
|
||||
// archive base 为 <dataDir>/file_baseline。path 为绝对路径。
|
||||
func ArchiveBeforeWrite(baseDir, path string) (archived bool, err error) {
|
||||
if !IsProtectedPath(path) {
|
||||
return false, nil
|
||||
}
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return false, nil // 新建文件无需留档
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
rel := strings.TrimPrefix(path, "/")
|
||||
dst := filepath.Join(filepath.Join(baseDir, "file_baseline"), rel)
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
// 已留档(内容未再变化时避免重复)
|
||||
if same, _ := sameContent(path, dst); same {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0600); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func sameContent(a, b string) (bool, error) {
|
||||
da, ea := os.ReadFile(a)
|
||||
if ea != nil {
|
||||
return false, ea
|
||||
}
|
||||
db, eb := os.ReadFile(b)
|
||||
if eb != nil {
|
||||
return false, eb
|
||||
}
|
||||
return string(da) == string(db), nil
|
||||
}
|
||||
|
||||
// FileBaselineDir 返回写前留档根目录。
|
||||
func FileBaselineDir(dataDir string) string {
|
||||
return filepath.Join(dataDir, "file_baseline")
|
||||
}
|
||||
|
||||
// RestoreFileFromBaseline 从写前留档恢复一个受保护文件(guard 使用)。
|
||||
// 若留档存在则写回并返回 true 与恢复路径;否则 false。
|
||||
func RestoreFileFromBaseline(dataDir, path string) (bool, error) {
|
||||
if !IsProtectedPath(path) {
|
||||
return false, nil
|
||||
}
|
||||
rel := strings.TrimPrefix(path, "/")
|
||||
dst := filepath.Join(FileBaselineDir(dataDir), rel)
|
||||
data, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
return false, fmt.Errorf("restore from baseline %s: %w", dst, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
240
internal/system/network.go
Normal file
240
internal/system/network.go
Normal file
@ -0,0 +1,240 @@
|
||||
// Package system 提供 L0/L1 层的系统级网络配置基线:捕获与还原本机的
|
||||
// DNS(resolv.conf)/ hosts / 代理环境,供 failback 在救援源恢复前还原被主 agent
|
||||
// 改坏的网络配置。
|
||||
package system
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NetworkBaseline 一张系统网络配置快照:捕获时点 + DNS/hosts/代理环境原文。
|
||||
type NetworkBaseline struct {
|
||||
CapturedAt string `json:"captured_at"`
|
||||
ResolvConf string `json:"resolv_conf,omitempty"` // /etc/resolv.conf 原文
|
||||
Hosts string `json:"hosts,omitempty"` // /etc/hosts 原文
|
||||
ProxyEnv map[string]string `json:"proxy_env,omitempty"` // http_proxy/https_proxy/no_proxy 等
|
||||
}
|
||||
|
||||
const baselineFileName = "network_baseline.json"
|
||||
|
||||
// relevantPath 本机网络相关文件,与代理环境变量一同纳入基线。
|
||||
// 默认值面向主流 Linux 发行版(systemd-resolved 等也写 /etc/resolv.conf)。
|
||||
// 若目标发行版布局不同(如 resolv.conf 在 /run/systemd/resolve/),由
|
||||
// SetNetworkPaths 在启动时注入,避免硬编码失效。
|
||||
var relevantPath = []string{
|
||||
"/etc/resolv.conf",
|
||||
"/etc/hosts",
|
||||
"/etc/environment",
|
||||
}
|
||||
|
||||
// SetNetworkPaths 覆盖纳入网络基线的文件路径列表(发行版适配入口)。
|
||||
// resolv.conf / hosts 用于还原与"被改动"判定;其余文件仅捕获(诊断用)。
|
||||
// 传 nil/空则恢复 Linux 默认。
|
||||
func SetNetworkPaths(paths []string) {
|
||||
if len(paths) == 0 {
|
||||
relevantPath = []string{"/etc/resolv.conf", "/etc/hosts", "/etc/environment"}
|
||||
return
|
||||
}
|
||||
ns := make([]string, 0, len(paths))
|
||||
for _, p := range paths {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
ns = append(ns, p)
|
||||
}
|
||||
}
|
||||
relevantPath = ns
|
||||
}
|
||||
|
||||
// NetworkPaths 返回当前纳入基线的网络文件路径(副本)。
|
||||
func NetworkPaths() []string {
|
||||
out := make([]string, len(relevantPath))
|
||||
copy(out, relevantPath)
|
||||
return out
|
||||
}
|
||||
|
||||
// envProxyKeys 纳入基线的代理相关环境变量(大小写归一,捕获时同时保存小写与大写形式)。
|
||||
var envProxyKeys = []string{
|
||||
"http_proxy", "https_proxy", "no_proxy",
|
||||
"HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY",
|
||||
"all_proxy", "ALL_PROXY",
|
||||
}
|
||||
|
||||
// fileKey 映射路径 → 基线字段。已知可还原字段:resolv_conf / hosts。
|
||||
func fileKey(p string) string {
|
||||
switch {
|
||||
case strings.Contains(p, "resolv"):
|
||||
return "resolv"
|
||||
case strings.Contains(p, "hosts"):
|
||||
return "hosts"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// CaptureNetwork 捕获当前系统网络配置成基线对象(不落盘)。
|
||||
func CaptureNetwork() (*NetworkBaseline, error) {
|
||||
b := &NetworkBaseline{
|
||||
CapturedAt: time.Now().Format(time.RFC3339),
|
||||
ProxyEnv: make(map[string]string),
|
||||
}
|
||||
for _, p := range relevantPath {
|
||||
if data, err := os.ReadFile(p); err == nil {
|
||||
switch fileKey(p) {
|
||||
case "resolv":
|
||||
b.ResolvConf = string(data)
|
||||
case "hosts":
|
||||
b.Hosts = string(data)
|
||||
default:
|
||||
if b.ProxyEnv == nil {
|
||||
b.ProxyEnv = make(map[string]string)
|
||||
}
|
||||
b.ProxyEnv["file:"+p] = string(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, k := range envProxyKeys {
|
||||
if v, ok := os.LookupEnv(k); ok {
|
||||
b.ProxyEnv[k] = v
|
||||
}
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// SaveNetworkBaseline 把网络基线写入 <dataDir>/network_baseline.json。
|
||||
func SaveNetworkBaseline(dataDir string, b *NetworkBaseline) error {
|
||||
path := NetworkBaselinePath(dataDir)
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
data, err := json.MarshalIndent(b, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, data, 0600)
|
||||
}
|
||||
|
||||
// LoadNetworkBaseline 从 <dataDir>/network_baseline.json 读回网络基线。
|
||||
func LoadNetworkBaseline(dataDir string) (*NetworkBaseline, error) {
|
||||
path := NetworkBaselinePath(dataDir)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var b NetworkBaseline
|
||||
if err := json.Unmarshal(data, &b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &b, nil
|
||||
}
|
||||
|
||||
// NetworkBaselinePath 返回网络基线文件路径。
|
||||
func NetworkBaselinePath(dataDir string) string {
|
||||
return filepath.Join(dataDir, "recovery", baselineFileName)
|
||||
}
|
||||
|
||||
// ApplyProxyEnv 把基线中的代理环境变量写回当前进程环境(guard 在拉起 worker 前调用,
|
||||
// 使 worker 继承干净代理)。
|
||||
func (b *NetworkBaseline) ApplyProxyEnv() {
|
||||
for _, k := range envProxyKeys {
|
||||
if v, ok := b.ProxyEnv[k]; ok {
|
||||
os.Setenv(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExtraProxyKeys 返回基线中额外捕获的键(如 ENVIRONMENT),排序后供诊断/报告使用。
|
||||
func (b *NetworkBaseline) ExtraProxyKeys() []string {
|
||||
var keys []string
|
||||
for k := range b.ProxyEnv {
|
||||
found := false
|
||||
for _, p := range envProxyKeys {
|
||||
if k == p {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(keys)
|
||||
return keys
|
||||
}
|
||||
|
||||
// restoreTargets 返回可还原的网络文件(键→基线内容)。由注入路径派生,
|
||||
// 适配不同发行版的 resolv.conf/hosts 位置。
|
||||
func (b *NetworkBaseline) restoreTargets() map[string]string {
|
||||
m := map[string]string{}
|
||||
for _, p := range relevantPath {
|
||||
switch fileKey(p) {
|
||||
case "resolv":
|
||||
if b.ResolvConf != "" {
|
||||
m[p] = b.ResolvConf
|
||||
}
|
||||
case "hosts":
|
||||
if b.Hosts != "" {
|
||||
m[p] = b.Hosts
|
||||
}
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// RestoreFiles 把基线中的 resolv.conf / hosts 写回原路径。仅当该路径被主 agent
|
||||
// 改动过(当前内容与基线不同)时才覆盖,返回改动项列表。
|
||||
func (b *NetworkBaseline) RestoreFiles() ([]string, error) {
|
||||
var changed []string
|
||||
for path, want := range b.restoreTargets() {
|
||||
cur, err := os.ReadFile(path)
|
||||
if err != nil || string(cur) == want {
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(want), 0644); err != nil {
|
||||
return changed, fmt.Errorf("restore %s: %w", path, err)
|
||||
}
|
||||
changed = append(changed, path)
|
||||
}
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// MismatchedFiles 返回与基线不一致的系统网络文件(guard 判定"网络是否被破坏")。
|
||||
// 返回名字列表;无则空。忽略读取失败的文件。
|
||||
func (b *NetworkBaseline) MismatchedFiles() []string {
|
||||
var out []string
|
||||
for path, want := range b.restoreTargets() {
|
||||
if cur, err := os.ReadFile(path); err != nil {
|
||||
out = append(out, path+"(unreadable)")
|
||||
} else if string(cur) != want {
|
||||
out = append(out, path+"(modified)")
|
||||
}
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsNetworkMismatch 判断主机网络配置与基线是否不一致(resolv.conf/hosts 中任一改动)。
|
||||
func (b *NetworkBaseline) IsNetworkMismatch() bool {
|
||||
return len(b.MismatchedFiles()) > 0
|
||||
}
|
||||
|
||||
// Summary 生成基线摘要(用于 failback 报告)。
|
||||
func (b *NetworkBaseline) Summary() string {
|
||||
parts := []string{fmt.Sprintf("captured=%s", b.CapturedAt)}
|
||||
if b.ResolvConf != "" {
|
||||
parts = append(parts, fmt.Sprintf("resolv=%dbytes", len(b.ResolvConf)))
|
||||
}
|
||||
if b.Hosts != "" {
|
||||
parts = append(parts, fmt.Sprintf("hosts=%dbytes", len(b.Hosts)))
|
||||
}
|
||||
if n := len(b.ProxyEnv); n > 0 {
|
||||
parts = append(parts, fmt.Sprintf("proxy_vars=%d", n))
|
||||
}
|
||||
return strings.Join(parts, " ")
|
||||
}
|
||||
162
internal/system/system_test.go
Normal file
162
internal/system/system_test.go
Normal file
@ -0,0 +1,162 @@
|
||||
package system
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestIsProtectedPath(t *testing.T) {
|
||||
cases := map[string]bool{
|
||||
"/etc/resolv.conf": true,
|
||||
"/etc/hosts": true,
|
||||
"/etc/apt/sources.list": true,
|
||||
"/etc/environment": true,
|
||||
"/tmp/foo.txt": false,
|
||||
"/home/user/x": false,
|
||||
"/usr/local/bin/homed": false,
|
||||
}
|
||||
for p, want := range cases {
|
||||
if got := IsProtectedPath(p); got != want {
|
||||
t.Errorf("IsProtectedPath(%q) = %v, want %v", p, got, want)
|
||||
}
|
||||
}
|
||||
// 发行版/部署路径注入:显式前缀集可扩展受保护范围
|
||||
SetProtectedPaths([]string{"/opt/llm-mock", "/home/newqqagent"})
|
||||
if !IsProtectedPath("/opt/llm-mock/mock_server.py") {
|
||||
t.Error("explicit prefix /opt/llm-mock should be protected")
|
||||
}
|
||||
if !IsProtectedPath("/home/newqqagent/config.yaml") {
|
||||
t.Error("explicit prefix /home/newqqagent should be protected")
|
||||
}
|
||||
SetProtectedPaths(nil) // 恢复默认
|
||||
if IsProtectedPath("/opt/llm-mock/mock_server.py") {
|
||||
t.Error("default should not protect /opt/llm-mock")
|
||||
}
|
||||
}
|
||||
|
||||
func TestArchiveBeforeWrite(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
// 受保护路径
|
||||
target := "/etc/ArchiveBeforeWrite.test.tmp"
|
||||
os.WriteFile(target, []byte("original"), 0644)
|
||||
defer os.Remove(target)
|
||||
|
||||
archived, err := ArchiveBeforeWrite(dir, target)
|
||||
if err != nil {
|
||||
t.Fatalf("ArchiveBeforeWrite: %v", err)
|
||||
}
|
||||
if !archived {
|
||||
t.Fatal("expected archive to happen")
|
||||
}
|
||||
dst := filepath.Join(dir, "file_baseline", strings.TrimPrefix(target, "/"))
|
||||
data, err := os.ReadFile(dst)
|
||||
if err != nil {
|
||||
t.Fatalf("read archived: %v", err)
|
||||
}
|
||||
if string(data) != "original" {
|
||||
t.Fatalf("archived content = %q, want original", data)
|
||||
}
|
||||
|
||||
// 幂等:同内容不重复
|
||||
archived2, err := ArchiveBeforeWrite(dir, target)
|
||||
if err != nil {
|
||||
t.Fatalf("ArchiveBeforeWrite #2: %v", err)
|
||||
}
|
||||
if archived2 {
|
||||
t.Fatal("expected idempotent archive (no repeat)")
|
||||
}
|
||||
|
||||
// 非保护路径不存档
|
||||
tmp := filepath.Join(t.TempDir(), "x.txt")
|
||||
os.WriteFile(tmp, []byte("x"), 0644)
|
||||
archived3, err := ArchiveBeforeWrite(dir, tmp)
|
||||
if err != nil {
|
||||
t.Fatalf("ArchiveBeforeWrite non-protected: %v", err)
|
||||
}
|
||||
if archived3 {
|
||||
t.Fatal("non-protected path should not archive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRestoreFileFromBaseline(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
target := "/etc/RestoreFileFromBaseline.test.tmp"
|
||||
os.WriteFile(target, []byte("v1"), 0644)
|
||||
defer os.Remove(target)
|
||||
|
||||
ArchiveBeforeWrite(dir, target)
|
||||
os.WriteFile(target, []byte("v2"), 0644)
|
||||
|
||||
restored, err := RestoreFileFromBaseline(dir, target)
|
||||
if err != nil {
|
||||
t.Fatalf("RestoreFileFromBaseline: %v", err)
|
||||
}
|
||||
if !restored {
|
||||
t.Fatal("expected restore to happen")
|
||||
}
|
||||
data, _ := os.ReadFile(target)
|
||||
if string(data) != "v1" {
|
||||
t.Fatalf("restored content = %q, want v1", data)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureNetworkBaselineRoundtrip(t *testing.T) {
|
||||
base, err := CaptureNetwork()
|
||||
if err != nil {
|
||||
t.Fatalf("CaptureNetwork: %v", err)
|
||||
}
|
||||
if base.CapturedAt == "" {
|
||||
t.Fatal("captured_at empty")
|
||||
}
|
||||
if base.ResolvConf == "" {
|
||||
t.Log("warning: no /etc/resolv.conf readable on this host")
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := SaveNetworkBaseline(dir, base); err != nil {
|
||||
t.Fatalf("SaveNetworkBaseline: %v", err)
|
||||
}
|
||||
loaded, err := LoadNetworkBaseline(dir)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadNetworkBaseline: %v", err)
|
||||
}
|
||||
if loaded.ResolvConf != base.ResolvConf {
|
||||
t.Fatal("resolv.conf roundtrip mismatch")
|
||||
}
|
||||
if loaded.CapturedAt != base.CapturedAt {
|
||||
t.Fatal("captured_at roundtrip mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMismatchedFiles(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
base := &NetworkBaseline{ResolvConf: "# baselinetest", Hosts: "# hosts"}
|
||||
// RestoreFiles 只写"当前内容与基线不同且基线非空"的文件;若 resolv.conf 恰好与
|
||||
// 测试用的假基线一致仍存在,则跳过。此处以空字段基线验证幂等(不破坏真实 /etc)。
|
||||
emptyBase := &NetworkBaseline{ResolvConf: "", Hosts: ""}
|
||||
if changed, err := emptyBase.RestoreFiles(); err != nil {
|
||||
t.Fatalf("RestoreFiles empty: %v", err)
|
||||
} else if len(changed) > 0 {
|
||||
t.Fatalf("empty baseline should change nothing, got %v", changed)
|
||||
}
|
||||
if base.Summary() == "" {
|
||||
t.Fatal("summary empty")
|
||||
}
|
||||
if _, err := LoadNetworkBaseline(dir); err == nil {
|
||||
t.Fatal("expected error loading missing baseline")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtraProxyKeys(t *testing.T) {
|
||||
base := &NetworkBaseline{ProxyEnv: map[string]string{
|
||||
"http_proxy": "http://p:8080",
|
||||
"HTTP_PROXY": "http://p:8080",
|
||||
"ENVIRONMENT": "FOO=bar",
|
||||
}}
|
||||
keys := base.ExtraProxyKeys()
|
||||
if len(keys) != 1 || keys[0] != "ENVIRONMENT" {
|
||||
t.Fatalf("ExtraProxyKeys = %v, want [ENVIRONMENT]", keys)
|
||||
}
|
||||
}
|
||||
@ -40,6 +40,9 @@ func fileHash(path string) (string, int64, error) {
|
||||
return hex.EncodeToString(h[:]), int64(len(data)), nil
|
||||
}
|
||||
|
||||
// maxCapturedContent 回滚内容捕获上限:超大文件不保存原文(回滚时跳过并告警)。
|
||||
const maxCapturedContent = 8 << 20
|
||||
|
||||
func fileInfo(path string) (size int64, modTime time.Time, err error) {
|
||||
info, err := os.Stat(path)
|
||||
if err != nil {
|
||||
@ -53,6 +56,7 @@ type FSState struct {
|
||||
Root string `json:"root"`
|
||||
}
|
||||
|
||||
// captureFSState 仅记录哈希/尺寸(用于"之后"快照,省内存)。
|
||||
func captureFSState(root string) (*FSState, error) {
|
||||
state := &FSState{
|
||||
Files: make(map[string]FileChange),
|
||||
@ -77,6 +81,27 @@ func captureFSState(root string) (*FSState, error) {
|
||||
return state, err
|
||||
}
|
||||
|
||||
// captureFSStateWithContent 额外捕获文件原文(用于"之前"基线,供回滚还原被改/被删文件)。
|
||||
func captureFSStateWithContent(root string) (*FSState, error) {
|
||||
state, err := captureFSState(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for rel := range state.Files {
|
||||
path := filepath.Join(root, rel)
|
||||
info, err := os.Stat(path)
|
||||
if err != nil || info.Size() > maxCapturedContent {
|
||||
continue
|
||||
}
|
||||
if data, err := os.ReadFile(path); err == nil {
|
||||
fc := state.Files[rel]
|
||||
fc.Content = data
|
||||
state.Files[rel] = fc
|
||||
}
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
func diffStates(before, after *FSState) []FileChange {
|
||||
var changes []FileChange
|
||||
if before == nil || after == nil {
|
||||
@ -95,6 +120,7 @@ func diffStates(before, after *FSState) []FileChange {
|
||||
HashAfter: afterFile.HashAfter,
|
||||
SizeBefore: beforeFile.SizeAfter,
|
||||
SizeAfter: afterFile.SizeAfter,
|
||||
Content: beforeFile.Content, // 原始内容,供回滚还原
|
||||
})
|
||||
}
|
||||
} else {
|
||||
@ -114,6 +140,7 @@ func diffStates(before, after *FSState) []FileChange {
|
||||
Type: ChangeFileDeleted,
|
||||
HashBefore: before.Files[path].HashAfter,
|
||||
SizeBefore: before.Files[path].SizeAfter,
|
||||
Content: before.Files[path].Content, // 原始内容,供回滚还原
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
114
internal/tracker/disk.go
Normal file
114
internal/tracker/disk.go
Normal file
@ -0,0 +1,114 @@
|
||||
package tracker
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// LoadChangeSetsFromDisk 把 <dataDir>/changesets/*.json 读回内存 changeSets
|
||||
// (按修改时间正序),使 guard 可在 worker 未运行时离线回滚 agentfs。
|
||||
func (t *Tracker) LoadChangeSetsFromDisk() (int, error) {
|
||||
dir := filepath.Join(t.dataDir, "changesets")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return 0, nil
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
|
||||
type csFile struct {
|
||||
path string
|
||||
mod time.Time
|
||||
}
|
||||
var files []csFile
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
|
||||
continue
|
||||
}
|
||||
info, err := e.Info()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
files = append(files, csFile{path: filepath.Join(dir, e.Name()), mod: info.ModTime()})
|
||||
}
|
||||
sort.Slice(files, func(i, j int) bool { return files[i].mod.Before(files[j].mod) })
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
t.changeSets = t.changeSets[:0]
|
||||
for _, f := range files {
|
||||
data, err := os.ReadFile(f.path)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var cs ChangeSet
|
||||
if err := json.Unmarshal(data, &cs); err != nil {
|
||||
continue
|
||||
}
|
||||
t.loadContentBlobs(&cs, dir)
|
||||
t.changeSets = append(t.changeSets, &cs)
|
||||
}
|
||||
log.Printf("[tracker] loaded %d changesets from disk", len(t.changeSets))
|
||||
return len(t.changeSets), nil
|
||||
}
|
||||
|
||||
// RollbackFromDisk 供 guard 在 worker 离线时执行 L2 agentfs 回滚:
|
||||
// 读回全部持久化 changeset 并按时间逆序逆应用(还原被改/被删文件、删除新增),
|
||||
// 然后删除这些 changeset 文件。返回还原的 changeset 数。
|
||||
func (t *Tracker) RollbackFromDisk() (int, error) {
|
||||
if _, err := t.LoadChangeSetsFromDisk(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
n := len(t.changeSets)
|
||||
if n == 0 {
|
||||
log.Printf("[tracker] rollback from disk: nothing to revert")
|
||||
return 0, nil
|
||||
}
|
||||
for i := n - 1; i >= 0; i-- {
|
||||
t.applyReverseLocked(t.changeSets[i])
|
||||
}
|
||||
dir := filepath.Join(t.dataDir, "changesets")
|
||||
for _, cs := range t.changeSets {
|
||||
os.Remove(filepath.Join(dir, cs.ID+".json"))
|
||||
removeContentBlobs(dir, cs.ID)
|
||||
}
|
||||
t.changeSets = t.changeSets[:0]
|
||||
log.Printf("[tracker] rollback from disk: reverted %d change sets", n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// ChangesetsOnDisk 返回磁盘上持久化 changeset 数量(guard 决策用)。
|
||||
func (t *Tracker) ChangesetsOnDisk() int {
|
||||
dir := filepath.Join(t.dataDir, "changesets")
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
n := 0
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() && strings.HasSuffix(e.Name(), ".json") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// NewOfflineTracker 构造一个仅用于离线回滚的 tracker(不 mount overlay)。
|
||||
// worker 目录不存在时也会自动创建(Init 语义)。
|
||||
func NewOfflineTracker(dataDir, workDir string) *Tracker {
|
||||
t := NewTracker(dataDir, workDir)
|
||||
_ = t.Init()
|
||||
return t
|
||||
}
|
||||
|
||||
var _ = fmt.Sprintf
|
||||
92
internal/tracker/disk_test.go
Normal file
92
internal/tracker/disk_test.go
Normal file
@ -0,0 +1,92 @@
|
||||
package tracker
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// 构造一条写入磁盘的 changeset(模拟 worker 运行期间的变更),随后用
|
||||
// RollbackFromDisk 离线还原。
|
||||
func TestRollbackFromDisk(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
workDir := t.TempDir()
|
||||
|
||||
trk := NewTracker(dataDir, workDir)
|
||||
if err := trk.Init(); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
upper := trk.upperDir
|
||||
os.MkdirAll(upper, 0755)
|
||||
|
||||
// 1. 先创建一个文件(对应"新建")
|
||||
newFile := filepath.Join(upper, "new.txt")
|
||||
os.WriteFile(newFile, []byte("brand new"), 0644)
|
||||
|
||||
// 2. 修改一个文件(对应"修改",带原文)
|
||||
modFile := filepath.Join(upper, "mod.txt")
|
||||
os.WriteFile(modFile, []byte("after"), 0644)
|
||||
beforeContent := []byte("before")
|
||||
csMod := &ChangeSet{
|
||||
ID: "cs_mod",
|
||||
Files: []FileChange{{
|
||||
Path: "mod.txt",
|
||||
Type: ChangeFileModified,
|
||||
Content: beforeContent,
|
||||
}},
|
||||
}
|
||||
trk.saveChangeSet(csMod)
|
||||
csNew := &ChangeSet{
|
||||
ID: "cs_new",
|
||||
Files: []FileChange{{
|
||||
Path: "new.txt",
|
||||
Type: ChangeFileCreated,
|
||||
}},
|
||||
}
|
||||
trk.saveChangeSet(csNew)
|
||||
|
||||
if trk.ChangesetsOnDisk() != 2 {
|
||||
t.Fatalf("ChangesetsOnDisk = %d, want 2", trk.ChangesetsOnDisk())
|
||||
}
|
||||
|
||||
// 离线回滚(模拟 guard 在 worker 崩溃后调用)
|
||||
n, err := trk.RollbackFromDisk()
|
||||
if err != nil {
|
||||
t.Fatalf("RollbackFromDisk: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("rolled back %d, want 2", n)
|
||||
}
|
||||
if trk.ChangesetsOnDisk() != 0 {
|
||||
t.Fatalf("ChangesetsOnDisk after rollback = %d, want 0", trk.ChangesetsOnDisk())
|
||||
}
|
||||
|
||||
// 新建文件被删除
|
||||
if _, err := os.Stat(newFile); !os.IsNotExist(err) {
|
||||
t.Fatal("created file should be removed after rollback")
|
||||
}
|
||||
// 修改文件还原原文
|
||||
data, err := os.ReadFile(modFile)
|
||||
if err != nil {
|
||||
t.Fatalf("read modFile: %v", err)
|
||||
}
|
||||
if string(data) != "before" {
|
||||
t.Fatalf("modFile restored to %q, want %q", data, beforeContent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackFromDiskEmpty(t *testing.T) {
|
||||
dataDir := t.TempDir()
|
||||
workDir := t.TempDir()
|
||||
trk := NewTracker(dataDir, workDir)
|
||||
if err := trk.Init(); err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
n, err := trk.RollbackFromDisk()
|
||||
if err != nil {
|
||||
t.Fatalf("RollbackFromDisk: %v", err)
|
||||
}
|
||||
if n != 0 {
|
||||
t.Fatalf("rolled back %d, want 0", n)
|
||||
}
|
||||
}
|
||||
@ -26,7 +26,7 @@ type Tracker struct {
|
||||
active bool
|
||||
before *FSState
|
||||
changeSets []*ChangeSet
|
||||
keepChangesets int // 保留最近 N 份 changeset,0 = 不限
|
||||
keepChangesets int // 保留最近 N 份 changeset,0 = 不限
|
||||
maxChangesetAge time.Duration // changeset 最大保留时长,0 = 不限
|
||||
}
|
||||
|
||||
@ -115,7 +115,7 @@ func (t *Tracker) PostAction(action string) *ChangeSet {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
after := t.capture()
|
||||
after, _ := captureFSState(t.upperDir)
|
||||
changes := diffStates(t.before, after)
|
||||
|
||||
cs := NewChangeSet(action)
|
||||
@ -146,7 +146,7 @@ func (t *Tracker) ChangeSets() []*ChangeSet {
|
||||
}
|
||||
|
||||
func (t *Tracker) capture() *FSState {
|
||||
state, err := captureFSState(t.upperDir)
|
||||
state, err := captureFSStateWithContent(t.upperDir)
|
||||
if err != nil {
|
||||
return &FSState{Files: make(map[string]FileChange), Root: t.upperDir}
|
||||
}
|
||||
@ -192,6 +192,41 @@ func (t *Tracker) saveChangeSet(cs *ChangeSet) {
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
log.Printf("[tracker] write changeset %s: %v", cs.ID, err)
|
||||
}
|
||||
|
||||
// Content 字段是 json:"-",不随 JSON 落盘;为保证 L2 离线回滚(guard 重启后)
|
||||
// 仍能还原被改/被删文件,把回滚原文作为伴随 blob 单独持久化。
|
||||
t.saveContentBlobs(cs, dir)
|
||||
}
|
||||
|
||||
// saveContentBlobs 把 changeset 中各文件的回滚原文写入 <dataDir>/changesets/<csID>_blobs/<i>.bin。
|
||||
func (t *Tracker) saveContentBlobs(cs *ChangeSet, dir string) {
|
||||
blobDir := filepath.Join(dir, cs.ID+"_blobs")
|
||||
os.MkdirAll(blobDir, 0755)
|
||||
for i, f := range cs.Files {
|
||||
if len(f.Content) == 0 {
|
||||
continue
|
||||
}
|
||||
bp := filepath.Join(blobDir, fmt.Sprintf("%03d.bin", i))
|
||||
if err := os.WriteFile(bp, f.Content, 0600); err != nil {
|
||||
log.Printf("[tracker] write blob %s: %v", bp, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadContentBlobs 读回 <csID>_blobs 目录中的回滚原文到 FileChange.Content。
|
||||
func (t *Tracker) loadContentBlobs(cs *ChangeSet, dir string) {
|
||||
blobDir := filepath.Join(dir, cs.ID+"_blobs")
|
||||
for i := range cs.Files {
|
||||
bp := filepath.Join(blobDir, fmt.Sprintf("%03d.bin", i))
|
||||
if data, err := os.ReadFile(bp); err == nil {
|
||||
cs.Files[i].Content = data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// removeContentBlobs 删除一个 changeset 的 blob 目录。
|
||||
func removeContentBlobs(dir, id string) {
|
||||
os.RemoveAll(filepath.Join(dir, id+"_blobs"))
|
||||
}
|
||||
|
||||
func (t *Tracker) cleanupChangeSets() {
|
||||
@ -202,8 +237,8 @@ func (t *Tracker) cleanupChangeSets() {
|
||||
}
|
||||
|
||||
type csFile struct {
|
||||
name string
|
||||
info os.FileInfo
|
||||
name string
|
||||
info os.FileInfo
|
||||
}
|
||||
var files []csFile
|
||||
for _, e := range entries {
|
||||
@ -243,6 +278,7 @@ func (t *Tracker) cleanupChangeSets() {
|
||||
for i := 0; i < excess; i++ {
|
||||
path := filepath.Join(dir, remaining[i].name)
|
||||
os.Remove(path)
|
||||
removeContentBlobs(dir, strings.TrimSuffix(remaining[i].name, ".json"))
|
||||
}
|
||||
remaining = remaining[excess:]
|
||||
}
|
||||
@ -253,6 +289,9 @@ func (t *Tracker) cleanupChangeSets() {
|
||||
}
|
||||
}
|
||||
|
||||
// Rollback 全量回滚:按时间逆序应用每个 changeset 的逆操作,把工作区恢复到
|
||||
// 首条 changeset 之前的状态(用捕获的原文还原被改/被删文件、删除新增文件)。
|
||||
// 无任何 changeset 时退回整目录重置(移除 upper 重建,丢弃全部变更)。
|
||||
func (t *Tracker) Rollback() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
@ -263,22 +302,77 @@ func (t *Tracker) Rollback() error {
|
||||
}
|
||||
}
|
||||
|
||||
if len(t.changeSets) > 0 {
|
||||
for i := len(t.changeSets) - 1; i >= 0; i-- {
|
||||
t.applyReverseLocked(t.changeSets[i])
|
||||
}
|
||||
log.Printf("[tracker] rollback complete: reverted %d change sets", len(t.changeSets))
|
||||
} else {
|
||||
if err := t.resetUpperLocked(); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("[tracker] rollback complete: no change sets, reset upper dir")
|
||||
}
|
||||
|
||||
t.changeSets = nil
|
||||
t.before = nil
|
||||
t.mounted = false
|
||||
return nil
|
||||
}
|
||||
|
||||
// RollbackLatest 仅撤销最近一条 changeset(定向回滚,不动更早的改动)。
|
||||
func (t *Tracker) RollbackLatest() error {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
|
||||
if len(t.changeSets) == 0 {
|
||||
return fmt.Errorf("no change sets to roll back")
|
||||
}
|
||||
cs := t.changeSets[len(t.changeSets)-1]
|
||||
t.applyReverseLocked(cs)
|
||||
t.changeSets = t.changeSets[:len(t.changeSets)-1]
|
||||
t.before = t.capture()
|
||||
log.Printf("[tracker] rolled back latest change set %s (%d files)", cs.ID, len(cs.Files))
|
||||
return nil
|
||||
}
|
||||
|
||||
// applyReverseLocked 逆应用一个 changeset:created→删除;modified→写回原文;deleted→用原文重建。
|
||||
// 调用方须持有写锁。
|
||||
func (t *Tracker) applyReverseLocked(cs *ChangeSet) {
|
||||
for _, f := range cs.Files {
|
||||
path := filepath.Join(t.upperDir, filepath.Clean(f.Path))
|
||||
switch f.Type {
|
||||
case ChangeFileCreated:
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
log.Printf("[tracker] rollback remove %s: %v", f.Path, err)
|
||||
}
|
||||
case ChangeFileModified, ChangeFileDeleted:
|
||||
if len(f.Content) == 0 {
|
||||
log.Printf("[tracker] rollback %s: original content not captured, skipping", f.Path)
|
||||
continue
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
|
||||
log.Printf("[tracker] rollback mkdir %s: %v", filepath.Dir(f.Path), err)
|
||||
continue
|
||||
}
|
||||
if err := os.WriteFile(path, f.Content, 0644); err != nil {
|
||||
log.Printf("[tracker] rollback restore %s: %v", f.Path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resetUpperLocked 整目录重置(无 changeset 时的兜底),调用方须持有写锁。
|
||||
func (t *Tracker) resetUpperLocked() error {
|
||||
if err := os.RemoveAll(t.upperDir); err != nil {
|
||||
return fmt.Errorf("remove upper: %w", err)
|
||||
}
|
||||
if err := os.RemoveAll(filepath.Join(t.workDir, "work")); err != nil {
|
||||
return fmt.Errorf("remove work: %w", err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(t.upperDir, 0755); err != nil {
|
||||
return fmt.Errorf("recreate upper: %w", err)
|
||||
}
|
||||
|
||||
t.changeSets = nil
|
||||
t.before = nil
|
||||
t.mounted = false
|
||||
|
||||
log.Printf("[tracker] rollback complete")
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@ -312,3 +312,134 @@ func TestCaptureDirNotExist(t *testing.T) {
|
||||
t.Error("expected error for nonexistent directory")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffStatesModifiedContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
f := filepath.Join(dir, "f.txt")
|
||||
os.WriteFile(f, []byte("original-content"), 0644)
|
||||
before, err := captureFSStateWithContent(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(f, []byte("tampered-content"), 0644)
|
||||
after, _ := captureFSState(dir)
|
||||
|
||||
changes := diffStates(before, after)
|
||||
if len(changes) != 1 || changes[0].Type != ChangeFileModified {
|
||||
t.Fatalf("expected 1 modified, got %+v", changes)
|
||||
}
|
||||
if string(changes[0].Content) != "original-content" {
|
||||
t.Fatalf("modified change should carry original content, got %q", changes[0].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiffStatesDeletedContent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
f := filepath.Join(dir, "f.txt")
|
||||
os.WriteFile(f, []byte("do-not-lose"), 0644)
|
||||
before, err := captureFSStateWithContent(dir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.Remove(f)
|
||||
after, _ := captureFSState(dir)
|
||||
|
||||
changes := diffStates(before, after)
|
||||
if len(changes) != 1 || changes[0].Type != ChangeFileDeleted {
|
||||
t.Fatalf("expected 1 deleted, got %+v", changes)
|
||||
}
|
||||
if string(changes[0].Content) != "do-not-lose" {
|
||||
t.Fatalf("deleted change should carry original content, got %q", changes[0].Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackLatest(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
work := filepath.Join(dir, "work")
|
||||
tr := NewTracker(filepath.Join(dir, "data"), work)
|
||||
if err := tr.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
upper := tr.upperDir
|
||||
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("stable"), 0644)
|
||||
os.WriteFile(filepath.Join(upper, "gone.txt"), []byte("do-not-lose"), 0644)
|
||||
|
||||
// 动作1:改 keep、加 new(应保留)
|
||||
tr.PreAction("action1")
|
||||
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("tampered"), 0644)
|
||||
os.WriteFile(filepath.Join(upper, "new.txt"), []byte("added"), 0644)
|
||||
tr.PostAction("action1")
|
||||
|
||||
// 动作2:删 gone(仅撤销这条)
|
||||
tr.PreAction("action2")
|
||||
os.Remove(filepath.Join(upper, "gone.txt"))
|
||||
tr.PostAction("action2")
|
||||
|
||||
if !tr.HasChanges() {
|
||||
t.Fatal("expected changes after PostAction")
|
||||
}
|
||||
|
||||
if err := tr.RollbackLatest(); err != nil {
|
||||
t.Fatalf("RollbackLatest: %v", err)
|
||||
}
|
||||
|
||||
restored, err := os.ReadFile(filepath.Join(upper, "gone.txt"))
|
||||
if err != nil || string(restored) != "do-not-lose" {
|
||||
t.Fatalf("gone.txt should be recreated with original content, got %q err=%v", restored, err)
|
||||
}
|
||||
// action1 的改动不受影响
|
||||
got, err := os.ReadFile(filepath.Join(upper, "keep.txt"))
|
||||
if err != nil || string(got) != "tampered" {
|
||||
t.Fatalf("keep.txt should keep action1 changes, got %q err=%v", got, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(upper, "new.txt")); err != nil {
|
||||
t.Fatalf("new.txt should still exist after latest-only rollback, err=%v", err)
|
||||
}
|
||||
if tr.HasChanges() != true {
|
||||
t.Fatal("earlier change sets should remain after latest-only rollback")
|
||||
}
|
||||
if got := len(tr.ChangeSets()); got != 1 {
|
||||
t.Fatalf("expected 1 remaining change set, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollbackFull(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
work := filepath.Join(dir, "work")
|
||||
tr := NewTracker(filepath.Join(dir, "data"), work)
|
||||
if err := tr.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
upper := tr.upperDir
|
||||
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("stable"), 0644)
|
||||
os.WriteFile(filepath.Join(upper, "deleteme.txt"), []byte("bye"), 0644)
|
||||
|
||||
// 动作1:改 keep、加 new
|
||||
tr.PreAction("a")
|
||||
os.WriteFile(filepath.Join(upper, "keep.txt"), []byte("tampered"), 0644)
|
||||
os.WriteFile(filepath.Join(upper, "new.txt"), []byte("added"), 0644)
|
||||
tr.PostAction("a")
|
||||
|
||||
// 动作2:删 deleteme(在受追踪的动作内)
|
||||
tr.PreAction("b")
|
||||
os.Remove(filepath.Join(upper, "deleteme.txt"))
|
||||
tr.PostAction("b")
|
||||
|
||||
if err := tr.Rollback(); err != nil {
|
||||
t.Fatalf("Rollback: %v", err)
|
||||
}
|
||||
|
||||
got, _ := os.ReadFile(filepath.Join(upper, "keep.txt"))
|
||||
if string(got) != "stable" {
|
||||
t.Fatalf("keep.txt should be restored, got %q", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(upper, "new.txt")); !os.IsNotExist(err) {
|
||||
t.Fatalf("new.txt should be gone after full rollback")
|
||||
}
|
||||
restored, err := os.ReadFile(filepath.Join(upper, "deleteme.txt"))
|
||||
if err != nil || string(restored) != "bye" {
|
||||
t.Fatalf("deleted file should be recreated with original content, got %q err=%v", restored, err)
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,46 +26,47 @@ type AgentID string
|
||||
type SnapshotID string
|
||||
|
||||
type Snapshot struct {
|
||||
ID SnapshotID `json:"id"`
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Reason string `json:"reason"`
|
||||
Size int64 `json:"size_bytes"`
|
||||
DockerImage string `json:"docker_image,omitempty"`
|
||||
Valid bool `json:"valid"`
|
||||
ID SnapshotID `json:"id"`
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Reason string `json:"reason"`
|
||||
Size int64 `json:"size_bytes"`
|
||||
DockerImage string `json:"docker_image,omitempty"`
|
||||
Valid bool `json:"valid"`
|
||||
}
|
||||
|
||||
type Heartbeat struct {
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
State AgentState `json:"state"`
|
||||
Health HealthStatus `json:"health"`
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
State AgentState `json:"state"`
|
||||
Health HealthStatus `json:"health"`
|
||||
Uptime time.Duration `json:"uptime"`
|
||||
LLMConnected bool `json:"llm_connected"`
|
||||
Error string `json:"error,omitempty"`
|
||||
LLMConnected bool `json:"llm_connected"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type NetworkCheckResult struct {
|
||||
LLMAPIReachable bool `json:"llm_api_reachable"`
|
||||
DNSResolving bool `json:"dns_resolving"`
|
||||
TCPReachable bool `json:"tcp_reachable"`
|
||||
Latency time.Duration `json:"latency_ms"`
|
||||
LatencyDegraded bool `json:"latency_degraded"`
|
||||
Error string `json:"error,omitempty"`
|
||||
LLMAPIReachable bool `json:"llm_api_reachable"`
|
||||
EndpointsConfigured bool `json:"endpoints_configured"`
|
||||
DNSResolving bool `json:"dns_resolving"`
|
||||
TCPReachable bool `json:"tcp_reachable"`
|
||||
Latency time.Duration `json:"latency_ms"`
|
||||
LatencyDegraded bool `json:"latency_degraded"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type SnapshotPolicy struct {
|
||||
Interval time.Duration `json:"interval"`
|
||||
MaxSnapshots int `json:"max_snapshots"`
|
||||
PreAction bool `json:"pre_action"`
|
||||
PostAction bool `json:"post_action"`
|
||||
Interval time.Duration `json:"interval"`
|
||||
MaxSnapshots int `json:"max_snapshots"`
|
||||
PreAction bool `json:"pre_action"`
|
||||
PostAction bool `json:"post_action"`
|
||||
}
|
||||
|
||||
type RollbackPolicy struct {
|
||||
MaxRetries int `json:"max_retries"`
|
||||
HealthThreshold HealthStatus `json:"health_threshold"`
|
||||
CooldownPeriod time.Duration `json:"cooldown_period"`
|
||||
AutoRollback bool `json:"auto_rollback"`
|
||||
MaxRetries int `json:"max_retries"`
|
||||
HealthThreshold HealthStatus `json:"health_threshold"`
|
||||
CooldownPeriod time.Duration `json:"cooldown_period"`
|
||||
AutoRollback bool `json:"auto_rollback"`
|
||||
}
|
||||
|
||||
type AgentConfig struct {
|
||||
@ -87,12 +88,12 @@ type ResourceLimit struct {
|
||||
}
|
||||
|
||||
type OperationLog struct {
|
||||
ID string `json:"id"`
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Action string `json:"action"`
|
||||
ID string `json:"id"`
|
||||
AgentID AgentID `json:"agent_id"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Action string `json:"action"`
|
||||
SnapshotID SnapshotID `json:"snapshot_id,omitempty"`
|
||||
Success bool `json:"success"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
|
||||
type LLMSource struct {
|
||||
|
||||
Reference in New Issue
Block a user