fix(deploy): 故障通知的三处缺陷 —— 死信目录、隔夜补投、失败无原因
起因：用户邮箱里「[dsh] 桥服务异常终止」反复出现。查下来有两层。
**第一层：崩溃本身（已修，是历史）**
dsh 在 2026-09-12 11:40 起崩溃循环，根因是当时那次插件快照切换后
`/root/.dsh/profiles/web/node_modules/dsh-mail-bridge/cordis.patch.yml` 不存在
（`failed to read overlay … ENOENT`）→ 起不来 → systemd 反复重启。
现在快照里该文件在、dsh `NRestarts=0`、今天 0 次失败。
**第二层：通知管线本身坏了（本次修）**
1. ★ **死信目录**：`zcode.service`（应用单元）既没有 `AGENTMAIL_AGENT_NAME` 也没有
密钥，报告就落进 `unknown-agent/` —— 而 flush **只读自己那个 Agent 的目录**，
于是 25 份 zcode 崩溃告警永久投不出去。修法是两件事：
· `resolveIdentity()`：身份按「单元 env → 单元名推导 → /etc/agentmail/<agent>.env」
解析；`zcode.service`→zcode、`pi-mail-bridge.service`→pi……
· 支持 `AGENTMAIL_AGENT_SECRET`：zcode 只配了 secret 没有 key，而脚本原先只认
Bearer key ⇒ 就算目录对了也发不出去（网关的 AgentAuth 两种都认）。
2. ★ **隔夜补投**：补投原先只在**同一个单元**的下次 `ExecStartPost` 跑，于是
网关不可达时攒下的报告要等到那个服务自己重启才补投 —— 实测 4 封 Sep-12 的
告警在 Sep-13 10:35 才到。现在新增 `agentmail-failure-flush.timer`（每 10 分钟
`--flush-all`），它遍历所有 Agent 目录、按目录名逐个解析身份后补投。
过时报告还会在主题与正文上标 **「补投：这是 N 分钟前的故障报告，不代表现在仍在
故障」**（原先正文里只有昨天的时间戳，读起来像刚崩）。
3. **补投失败只报数不报因**：`catch { failed++ }` → 日志只有 `spool: sent=0 failed=4`，
没人知道卡在哪。现在每条失败都带回原因（HTTP 状态码/网关不可达/身份未配置）。
4. 顺带两处准确性问题：
· 主题写**单元名**而不是笼统的「桥服务」—— 25 封标题写着"桥服务异常终止"，
实际崩的是 zcode **应用**单元，照标题去查桥方向就错了。
· `created_at_ms` 是我们的元数据，但 `/mail/send` 是**严格解码**的（实测 400
不认识的字段）→ 发送前剥离，线格式保持干净。
判据：新增 `deploy/service-failure-notify.test.mjs`（12 条，含假网关做真实投递、
严格解码断言、补投标记的正反两向）。端到端验证：模拟"没有身份的 zcode.service
崩溃"——修复前落 `unknown-agent/` 永久死信；现在**真的投出去了**，且
`from_name=zcode`、主题 `[zcode] zcode.service 异常终止 #…`。
积压清理：27 份（dsh 2 + unknown-agent 25）全部来自 09-12 那两轮崩溃、事件已在
邮箱与 journal 里出现过，按**不再补投**处理（避免把隔夜告警灌进邮箱），
原始文件留档 /root/gotmp/failure-spool-backlog-20260913.tar.gz（600），
spool 目录留 README.md 说明。
This commit is contained in:
14
deploy/agentmail-failure-flush.service
Normal file
14
deploy/agentmail-failure-flush.service
Normal file
@ -0,0 +1,14 @@
|
||||
# 定期把 spool 里的故障通知补投出去（配合 agentmail-failure-flush.timer）。
|
||||
#
|
||||
# 为什么需要它：补投原先只在**同一个单元**的 ExecStartPost 跑 `--flush`，
|
||||
# 于是两条路都堵死 —— 报告在某 Agent 目录里而它一直没重启（实测隔一天才补投），
|
||||
# 或报告落进 unknown-agent/（**任何** flush 都不读那个目录，实测 25 封死信）。
|
||||
# 定时跑 --flush-all 让这两种情形都不再发生：它遍历所有 Agent 目录，
|
||||
# 按目录名从 /etc/agentmail/<agent>.env 解析各自的身份再投。
|
||||
[Unit]
|
||||
Description=AgentMail 故障通知补投（排空 spool）
|
||||
Documentation=file:///home/program/agentmail/docs/PLAN.md
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/bin/node /home/program/agentmail/deploy/service-failure-notify.mjs --flush-all
|
||||
10
deploy/agentmail-failure-flush.timer
Normal file
10
deploy/agentmail-failure-flush.timer
Normal file
@ -0,0 +1,10 @@
|
||||
[Unit]
|
||||
Description=每 10 分钟补投 AgentMail 故障通知
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=10min
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@ -16,6 +16,7 @@
|
||||
*/
|
||||
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { basename, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@ -23,6 +24,19 @@ import { fileURLToPath } from 'node:url';
|
||||
const DEFAULT_GATEWAY = 'http://127.0.0.1:8180';
|
||||
const DEFAULT_RECIPIENT = 'jianf@';
|
||||
const DEFAULT_SPOOL = '/var/lib/agentmail/failure-spool';
|
||||
/**
|
||||
* Agent 身份文件目录。单元自己的 env 里未必有密钥（`zcode.service` 就没有 ——
|
||||
* 它是应用单元、不是桥），但同一个 Agent 的密钥一定在 `/etc/agentmail/<name>.env`。
|
||||
*/
|
||||
const DEFAULT_ENV_DIR = '/etc/agentmail';
|
||||
/**
|
||||
* 早于这个时长的报告在补投时会被标成「补投」。
|
||||
*
|
||||
* 为什么必须标：实测有 4 封 Sep-12 的崩溃告警因为网关当时不可达而落 spool，
|
||||
* 直到第二天重启服务才补投出去 —— 正文里的时间戳是**昨天**，而人打开邮箱时
|
||||
* 会以为服务刚刚又崩了。告警的时效性就是它的全部价值。
|
||||
*/
|
||||
const STALE_MS = 30 * 60 * 1000;
|
||||
const MAX_SPOOL_FILES = 100;
|
||||
const REQUEST_TIMEOUT_MS = 5_000;
|
||||
|
||||
@ -87,7 +101,9 @@ export function buildFailurePayload({
|
||||
error = null,
|
||||
now = new Date(),
|
||||
} = {}) {
|
||||
const agentName = String(env.AGENTMAIL_AGENT_NAME || 'unknown-agent').trim();
|
||||
// 身份优先按单元名推导 + 从 /etc/agentmail/<agent>.env 取 —— 单元自己的 env
|
||||
// 里未必有名字（zcode.service 就没有），而正文与目录都要用它。
|
||||
const agentName = resolveIdentity(env, serviceName).name;
|
||||
const service = cleanServiceName(serviceName || env.AGENTMAIL_SERVICE_NAME, agentName);
|
||||
const result = String(env.SERVICE_RESULT || reason || 'unexpected-exit');
|
||||
const exitCode = String(env.EXIT_CODE || 'unknown');
|
||||
@ -112,6 +128,8 @@ export function buildFailurePayload({
|
||||
|
||||
return {
|
||||
to: String(env.AGENTMAIL_FAILURE_RECIPIENT || DEFAULT_RECIPIENT),
|
||||
// 机器可读的生成时刻：补投时据此判断这份报告有多旧（正文里的时间是给人看的）
|
||||
created_at_ms: now.getTime(),
|
||||
/*
|
||||
* 主题带上本次启动的 invocation 短号。
|
||||
*
|
||||
@ -128,29 +146,49 @@ export function buildFailurePayload({
|
||||
* 副作用是崩溃循环会产生多条会话而不是一条线程；对"服务在崩"这件事，
|
||||
* 分开计数比合并成一条更容易发现问题。
|
||||
*/
|
||||
subject: `[${agentName}] 桥服务异常终止 #${invocationTag(env)}`,
|
||||
// 主题里写**单元名**而不是笼统的"桥服务"：同一个 Agent 可能既有应用单元
|
||||
// （zcode.service）又有桥单元（zcode-mail-bridge.service），崩的常常是前者。
|
||||
// 实测代价：25 封标题写着"桥服务异常终止"，实际是 zcode **应用**单元在崩 ——
|
||||
// 照标题去查桥，方向从一开始就错了。
|
||||
subject: `[${agentName}] ${service} 异常终止 #${invocationTag(env)}`,
|
||||
body: lines.join('\n'),
|
||||
relay: 'summary',
|
||||
relay_key: relayKeyFor(env, service),
|
||||
};
|
||||
}
|
||||
|
||||
async function postPayload(payload, env = process.env) {
|
||||
const key = String(env.AGENTMAIL_AGENT_KEY || '').trim();
|
||||
if (!key) throw new Error('AGENTMAIL_AGENT_KEY 未配置');
|
||||
async function postPayload(payload, env = process.env, serviceName = '') {
|
||||
const identity = resolveIdentity(env, serviceName);
|
||||
if (!identity.key && !identity.secret) {
|
||||
throw new Error(
|
||||
`Agent 身份未配置（单元 env 与 /etc/agentmail/${identity.name}.env 里都没有 ` +
|
||||
'AGENTMAIL_AGENT_KEY / AGENTMAIL_AGENT_SECRET）'
|
||||
);
|
||||
}
|
||||
|
||||
const base = String(env.AGENTMAIL_GATEWAY_URL || DEFAULT_GATEWAY).replace(/\/+$/, '');
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
||||
try {
|
||||
/*
|
||||
* 剥掉我们自己的元数据再发。
|
||||
*
|
||||
* `/mail/send` 是**严格解码**的（不认识的字段直接 400），而 `created_at_ms`
|
||||
* 只是 spool 记录用来判断"这份报告有多旧"的，不属于线上格式。
|
||||
* 实测：带上它 → `400 不认识的字段 "created_at_ms"`，报告只好又回 spool。
|
||||
*/
|
||||
const { created_at_ms: _omit, ...wire } = payload;
|
||||
|
||||
const res = await fetch(`${base}/api/v1/mail/send`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${key}`,
|
||||
'X-Agent-Name': String(env.AGENTMAIL_AGENT_NAME || ''),
|
||||
// 两种身份都支持：有 key 走 Bearer，只有 secret 走 X-Agent-Name/Secret
|
||||
...(identity.key
|
||||
? { Authorization: `Bearer ${identity.key}` }
|
||||
: { 'X-Agent-Name': identity.name, 'X-Agent-Secret': identity.secret }),
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
body: JSON.stringify(wire),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const responseBody = await res.text();
|
||||
@ -167,8 +205,66 @@ function safeAgentName(env) {
|
||||
.slice(0, 96);
|
||||
}
|
||||
|
||||
function spoolDirFor(env) {
|
||||
return join(String(env.AGENTMAIL_FAILURE_SPOOL || DEFAULT_SPOOL), safeAgentName(env));
|
||||
/** 从 systemd 单元名推导 Agent 名：`zcode.service`→zcode、`pi-mail-bridge.service`→pi。 */
|
||||
export function agentNameFromService(serviceName) {
|
||||
const base = basename(String(serviceName || '').trim())
|
||||
.replace(/\.service$/, '')
|
||||
.trim();
|
||||
if (!base) return '';
|
||||
return base
|
||||
.replace(/-mail-bridge$/, '')
|
||||
.replace(/-bridge$/, '')
|
||||
.replace(/-serve$/, '')
|
||||
.replace(/-mail$/, '')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** 从 `/etc/agentmail/<agent>.env` 里读 `AGENTMAIL_AGENT_KEY`（systemd EnvironmentFile 格式）。 */
|
||||
export function readAgentSecretFromFile(file, varName = 'AGENTMAIL_AGENT_KEY') {
|
||||
try {
|
||||
const text = readFileSync(file, 'utf8');
|
||||
const m = text.match(new RegExp(`^\\s*${varName}\\s*=\\s*(.*)$`, 'm'));
|
||||
if (!m) return '';
|
||||
return String(m[1]).trim().replace(/^["']|["']$/g, '');
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析这次上报/补投该用哪个 Agent 身份。
|
||||
*
|
||||
* 单元自己的 env 优先；缺失时按单元名推导、再从 `/etc/agentmail/<name>.env` 取密钥。
|
||||
*
|
||||
* 为什么需要这一步（实测故障）：`zcode.service`（应用单元）崩溃时，它的 env 里
|
||||
* 既没有 `AGENTMAIL_AGENT_NAME` 也没有 `AGENTMAIL_AGENT_KEY`，报告就落进了
|
||||
* `unknown-agent/` 目录并一直待在那里 —— 而 **flush 只读自己那个 Agent 的目录**，
|
||||
* 于是 25 份 zcode 崩溃告警永久死信（它们在 2026-09-12 14:40 那轮崩溃循环里产生）。
|
||||
* 「报不出去」本身还能看见，**报不出去还落进没人读的目录**才是真的丢。
|
||||
*/
|
||||
export function resolveIdentity(env = process.env, serviceName = '') {
|
||||
const explicit = String(env.AGENTMAIL_AGENT_NAME || '').trim();
|
||||
const derived = agentNameFromService(serviceName || env.AGENTMAIL_SERVICE_NAME || '');
|
||||
const rawName = explicit || derived || 'unknown-agent';
|
||||
const name = rawName.replace(/[^A-Za-z0-9_.@-]/g, '-').slice(0, 96);
|
||||
|
||||
let key = String(env.AGENTMAIL_AGENT_KEY || '').trim();
|
||||
let secret = String(env.AGENTMAIL_AGENT_SECRET || '').trim();
|
||||
let keyFrom = '';
|
||||
if ((!key || !secret) && name !== 'unknown-agent') {
|
||||
const file = join(String(env.AGENTMAIL_ENV_DIR || DEFAULT_ENV_DIR), `${name}.env`);
|
||||
if (!key) key = readAgentSecretFromFile(file, 'AGENTMAIL_AGENT_KEY');
|
||||
// zcode 这类单元只配了 secret 没有 key，而网关的 AgentAuth 两种都认。
|
||||
// 脚本原先只认 key ⇒ zcode.service 的告警永远发不出去（实测 25 封死信）。
|
||||
if (!secret) secret = readAgentSecretFromFile(file, 'AGENTMAIL_AGENT_SECRET');
|
||||
if (key || secret) keyFrom = file;
|
||||
}
|
||||
return { name, key, secret, keyFrom, derived: !explicit && !!derived };
|
||||
}
|
||||
|
||||
function spoolDirFor(env, serviceName = '') {
|
||||
const { name } = resolveIdentity(env, serviceName);
|
||||
return join(String(env.AGENTMAIL_FAILURE_SPOOL || DEFAULT_SPOOL), name);
|
||||
}
|
||||
|
||||
function safeSpoolName(payload) {
|
||||
@ -190,10 +286,14 @@ async function pruneSpool(spoolDir) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function spoolPayload(payload, env = process.env) {
|
||||
export async function spoolPayload(payload, env = process.env, serviceName = '') {
|
||||
// 每个 Agent 独立目录：否则 pi 启动时可能拿自己的密钥去补发 dsh 的报告，
|
||||
// 发件身份与正文主体会错位，relay_key 的幂等范围也变了。
|
||||
const spoolDir = spoolDirFor(env);
|
||||
//
|
||||
// 目录名走 resolveIdentity（env → 单元名推导 → /etc/agentmail/<name>.env），
|
||||
// 否则没有 AGENTMAIL_AGENT_NAME 的单元（zcode.service 就是）会把报告落进
|
||||
// `unknown-agent/` —— 而 flush 只读自己那个目录，等于永久死信（实测 25 封）。
|
||||
const spoolDir = spoolDirFor(env, serviceName);
|
||||
await mkdir(spoolDir, { recursive: true, mode: 0o700 });
|
||||
const target = join(spoolDir, safeSpoolName(payload));
|
||||
const temp = `${target}.${process.pid}.${randomUUID()}.tmp`;
|
||||
@ -213,35 +313,70 @@ export async function notifyCrash(reason, error, options = {}) {
|
||||
now: options.now,
|
||||
});
|
||||
try {
|
||||
await postPayload(payload, env);
|
||||
await postPayload(payload, env, options.serviceName || '');
|
||||
return { sent: true, spooled: false, payload };
|
||||
} catch (sendError) {
|
||||
if (options.spool === false) throw sendError;
|
||||
const path = await spoolPayload(payload, env);
|
||||
const path = await spoolPayload(payload, env, options.serviceName || '');
|
||||
return { sent: false, spooled: true, path, payload, error: sendError };
|
||||
}
|
||||
}
|
||||
|
||||
export async function flushSpool(env = process.env) {
|
||||
const spoolDir = spoolDirFor(env);
|
||||
/**
|
||||
* 补投 spool 里的报告。
|
||||
*
|
||||
* 两个改动都是踩出来的：
|
||||
* - **失败要说原因**。原先是 `catch { failed++ }`，只报数不报因：4 封 dsh 告警
|
||||
* 在 spool 里躺了一整天，日志只有 `spool: sent=0 failed=N`，没人知道卡在哪。
|
||||
* - **过时的报告要标注**。Sep-12 的崩溃告警 Sep-13 才补投成功，正文时间是昨天，
|
||||
* 收信人只会以为服务刚刚又崩了。
|
||||
*/
|
||||
export async function flushSpool(env = process.env, serviceName = '') {
|
||||
const spoolDir = spoolDirFor(env, serviceName);
|
||||
const names = (await readdir(spoolDir).catch(() => []))
|
||||
.filter((name) => name.endsWith('.json'))
|
||||
.sort();
|
||||
let sent = 0;
|
||||
let failed = 0;
|
||||
const errors = [];
|
||||
|
||||
for (const name of names) {
|
||||
const path = join(spoolDir, name);
|
||||
try {
|
||||
const record = JSON.parse(await readFile(path, 'utf8'));
|
||||
await postPayload(record.payload, env);
|
||||
const payload = await markStaleIfNeeded(record.payload, path);
|
||||
await postPayload(payload, env, serviceName);
|
||||
await rm(path, { force: true });
|
||||
sent++;
|
||||
} catch {
|
||||
} catch (error) {
|
||||
failed++;
|
||||
errors.push({ file: name, reason: describeSendError(error) });
|
||||
}
|
||||
}
|
||||
return { total: names.length, sent, failed };
|
||||
return { total: names.length, sent, failed, errors, spoolDir };
|
||||
}
|
||||
|
||||
/**
|
||||
* 补投一份**已经过时**的报告时，把这件事写在正文最前面。
|
||||
*
|
||||
* 不标的后果（实测）：Sep-12 的崩溃告警在 Sep-13 才补投成功，正文里的时间是
|
||||
* 昨天，而收信人打开邮箱只会以为服务刚刚又崩了 —— 误导性的告警比没有更糟。
|
||||
*/
|
||||
async function markStaleIfNeeded(payload, filePath) {
|
||||
const createdMs =
|
||||
Number(payload?.created_at_ms || 0) ||
|
||||
(await stat(filePath).catch(() => null))?.mtimeMs ||
|
||||
0;
|
||||
if (!createdMs) return payload;
|
||||
const age = Date.now() - createdMs;
|
||||
if (age <= STALE_MS) return payload;
|
||||
const minutes = Math.round(age / 60000);
|
||||
const head = [
|
||||
`> **补投**：这是 ${minutes} 分钟前的故障报告（当时网关不可达或上报被拒，已暂存）。`,
|
||||
`> 它**不代表现在仍在故障**；服务下次启动时已触发补投。`,
|
||||
'',
|
||||
].join('\n');
|
||||
return { ...payload, subject: `[补投] ${payload.subject}`, body: `${head}${payload.body}` };
|
||||
}
|
||||
|
||||
/**
|
||||
@ -266,11 +401,57 @@ function describeSendError(error) {
|
||||
|
||||
async function main(argv = process.argv.slice(2), env = process.env) {
|
||||
const serviceName = argValue(argv, '--service');
|
||||
|
||||
/*
|
||||
* `--flush-all`：把 spool 下**所有** Agent 目录都排空一次。
|
||||
*
|
||||
* 为什么要它：`--flush` 只读自己那个 Agent 的目录，于是两条路都堵死 ——
|
||||
* ① 报告在 A 的目录里、而 A 一直没重启 → 隔一天才补投（实测 4 封）；
|
||||
* ② 报告落进 `unknown-agent/`（单元没给身份）→ **任何** flush 都不会读它
|
||||
* （实测 25 份 zcode 崩溃告警永久死信）。
|
||||
* 由定时器每 10 分钟跑一次这一条，上面两种情形都不再发生。
|
||||
*
|
||||
* 身份仍按目录逐个解析（目录名 = Agent 名 → /etc/agentmail/<name>.env 取密钥），
|
||||
* 所以不会出现「pi 拿自己的密钥去发 dsh 的报告」那种错位。
|
||||
*/
|
||||
if (argv.includes('--flush-all')) {
|
||||
const root = String(env.AGENTMAIL_FAILURE_SPOOL || DEFAULT_SPOOL);
|
||||
const dirs = (await readdir(root).catch(() => [])).filter((n) => !n.startsWith('.'));
|
||||
let sent = 0;
|
||||
let failed = 0;
|
||||
const errors = [];
|
||||
for (const dir of dirs) {
|
||||
const named = dir !== 'unknown-agent' && /^[A-Za-z0-9_.@-]+$/.test(dir);
|
||||
const sub = { ...env, AGENTMAIL_AGENT_NAME: named ? dir : '', AGENTMAIL_AGENT_KEY: '' };
|
||||
const result = await flushSpool(sub, named ? dir : '');
|
||||
sent += result.sent;
|
||||
failed += result.failed;
|
||||
for (const e of result.errors) errors.push({ dir, ...e });
|
||||
if (!named && result.total > 0) {
|
||||
console.error(
|
||||
`[agentmail-failure-notify] \u2605 ${dir}/ 有 ${result.total} 份报告但无法确定发件身份` +
|
||||
`（缺 ${DEFAULT_ENV_DIR}/<agent>.env）—— 它们永远发不出去，请修单位的 AGENTMAIL_AGENT_NAME`
|
||||
);
|
||||
}
|
||||
}
|
||||
if (sent || failed) {
|
||||
console.error(`[agentmail-failure-notify] flush-all: sent=${sent} failed=${failed}`);
|
||||
}
|
||||
for (const e of errors.slice(0, 10)) {
|
||||
console.error(`[agentmail-failure-notify] 补投失败 ${e.dir}/${String(e.file).slice(0, 12)}…：${e.reason}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (argv.includes('--flush')) {
|
||||
const result = await flushSpool(env);
|
||||
const result = await flushSpool(env, serviceName);
|
||||
if (result.sent || result.failed) {
|
||||
console.error(`[agentmail-failure-notify] spool: sent=${result.sent} failed=${result.failed}`);
|
||||
}
|
||||
// 只报数不报因的话，"卡了一天"这件事在日志里就是一句沉默（实测过）
|
||||
for (const e of result.errors) {
|
||||
console.error(`[agentmail-failure-notify] 补投失败 ${String(e.file).slice(0, 12)}…：${e.reason}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!argv.includes('--report')) throw new Error('需要 --report 或 --flush');
|
||||
|
||||
237
deploy/service-failure-notify.test.mjs
Normal file
237
deploy/service-failure-notify.test.mjs
Normal file
@ -0,0 +1,237 @@
|
||||
/**
|
||||
* 服务故障通知脚本的判据。
|
||||
*
|
||||
* 每一条都对应一次**实测故障**（见各 case 的注释），不是凭想象加的：
|
||||
* · zcode.service 的告警落进 `unknown-agent/` 且永远发不出去（25 份死信）
|
||||
* · 隔夜补投的报告读起来像"刚刚又崩了"
|
||||
* · 补投失败只报数不报因（`spool: sent=0 failed=4`，没人知道卡在哪）
|
||||
*/
|
||||
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { createServer } from 'node:http';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, readdirSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
|
||||
import {
|
||||
agentNameFromService,
|
||||
readAgentSecretFromFile,
|
||||
resolveIdentity,
|
||||
buildFailurePayload,
|
||||
spoolPayload,
|
||||
flushSpool,
|
||||
isAbnormalExit
|
||||
} from './service-failure-notify.mjs';
|
||||
|
||||
const tmp = () => mkdtempSync(join(tmpdir(), 'sfn-'));
|
||||
|
||||
/** 起一个假网关，记录收到的请求体，返回 200。 */
|
||||
async function fakeGateway(handler = null) {
|
||||
const got = [];
|
||||
const server = createServer((req, res) => {
|
||||
let body = '';
|
||||
req.on('data', c => (body += c));
|
||||
req.on('end', () => {
|
||||
const status = handler ? handler({ headers: req.headers, body }, got) : 200;
|
||||
got.push({ headers: req.headers, body });
|
||||
res.writeHead(status, { 'Content-Type': 'application/json' });
|
||||
res.end('{"ok":true}');
|
||||
});
|
||||
});
|
||||
await new Promise(r => server.listen(0, '127.0.0.1', r));
|
||||
const port = server.address().port;
|
||||
return { got, url: `http://127.0.0.1:${port}`, close: () => new Promise(r => server.close(r)) };
|
||||
}
|
||||
|
||||
// ─── 身份推导 ────────────────────────────────────────────────────────
|
||||
|
||||
test('★ 从单元名推导 Agent 名（zcode.service 不再落 unknown-agent）', () => {
|
||||
const cases = {
|
||||
'dsh.service': 'dsh',
|
||||
'zcode.service': 'zcode',
|
||||
'zcode-mail-bridge.service': 'zcode',
|
||||
'pi-mail-bridge.service': 'pi',
|
||||
'opencode-serve.service': 'opencode',
|
||||
'homeagent.service': 'homeagent'
|
||||
};
|
||||
for (const [unit, want] of Object.entries(cases)) {
|
||||
assert.equal(agentNameFromService(unit), want, unit);
|
||||
}
|
||||
assert.equal(agentNameFromService('/etc/systemd/system/dsh.service'), 'dsh');
|
||||
assert.equal(agentNameFromService(''), '');
|
||||
});
|
||||
|
||||
test('★ 只有 secret 的单元也能解析出身份（zcode 就是这种）', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sk-zcode-test\n');
|
||||
const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir }, 'zcode.service');
|
||||
assert.equal(id.name, 'zcode');
|
||||
assert.equal(id.key, '');
|
||||
assert.equal(id.secret, 'sk-zcode-test', '必须能从 <agent>.env 里读出 secret');
|
||||
});
|
||||
|
||||
test('单元自己的 env 优先于推导与文件', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'dsh.env'), 'AGENTMAIL_AGENT_KEY=from-file\n');
|
||||
const id = resolveIdentity({ AGENTMAIL_ENV_DIR: dir, AGENTMAIL_AGENT_NAME: 'pi', AGENTMAIL_AGENT_KEY: 'k1' }, 'dsh.service');
|
||||
assert.equal(id.name, 'pi');
|
||||
assert.equal(id.key, 'k1');
|
||||
assert.equal(id.derived, false);
|
||||
});
|
||||
|
||||
test('两种变量名都能读，引号会被剥掉', () => {
|
||||
const dir = tmp();
|
||||
writeFileSync(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY="q1"\n');
|
||||
writeFileSync(join(dir, 'b.env'), "AGENTMAIL_AGENT_SECRET='q2'\n");
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'a.env'), 'AGENTMAIL_AGENT_KEY'), 'q1');
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'b.env'), 'AGENTMAIL_AGENT_SECRET'), 'q2');
|
||||
assert.equal(readAgentSecretFromFile(join(dir, 'missing.env')), '');
|
||||
});
|
||||
|
||||
// ─── payload ─────────────────────────────────────────────────────────
|
||||
|
||||
test('payload 带机器可读时间与 invocation 短号（补投判据靠它）', () => {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', INVOCATION_ID: 'abc123def456', SERVICE_RESULT: 'exit-code', EXIT_CODE: 'exited', EXIT_STATUS: '1' },
|
||||
serviceName: 'dsh.service',
|
||||
now: new Date('2026-09-12T03:42:35.000Z')
|
||||
});
|
||||
assert.match(p.subject, /^\[dsh\] /);
|
||||
assert.match(p.subject, /#abc123de/);
|
||||
assert.equal(p.created_at_ms, Date.parse('2026-09-12T03:42:35.000Z'));
|
||||
assert.match(p.body, /2026-09-12T03:42:35/);
|
||||
});
|
||||
|
||||
test('secret 会被脱敏', () => {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'z', AGENTMAIL_AGENT_SECRET: 'super-secret-token', SERVICE_RESULT: 'exit-code' },
|
||||
error: new Error('boom super-secret-token'),
|
||||
serviceName: 'z.service'
|
||||
});
|
||||
assert.ok(!p.body.includes('super-secret-token'));
|
||||
});
|
||||
|
||||
test('正常退出不算异常', () => {
|
||||
assert.equal(isAbnormalExit({ SERVICE_RESULT: 'success' }), false);
|
||||
assert.equal(isAbnormalExit({ SERVICE_RESULT: 'exit-code' }), true);
|
||||
assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '0' }), false);
|
||||
assert.equal(isAbnormalExit({ EXIT_CODE: 'exited', EXIT_STATUS: '1' }), true);
|
||||
});
|
||||
|
||||
// ─── spool 落点 ──────────────────────────────────────────────────────
|
||||
|
||||
test('★ 没给 AGENTMAIL_AGENT_NAME 时，报告要落进推导出的 Agent 目录（不是 unknown-agent）', async () => {
|
||||
const root = tmp();
|
||||
const payload = buildFailurePayload({
|
||||
env: { SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'zcode.service'
|
||||
});
|
||||
await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, 'zcode.service');
|
||||
assert.deepEqual(readdirSync(root), ['zcode'], '落错目录 = 没有任何 flush 会读它（实测 25 份死信）');
|
||||
// 反向对照：完全无身份信息时只能落 unknown-agent（并且 flush-all 会警告）
|
||||
await spoolPayload(payload, { AGENTMAIL_FAILURE_SPOOL: root }, '');
|
||||
assert.ok(readdirSync(root).includes('unknown-agent'));
|
||||
});
|
||||
|
||||
// ─── 补投 ────────────────────────────────────────────────────────────
|
||||
|
||||
test('★ 补投陈旧报告要标「补投」并说明不代表现在仍故障', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const old = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service',
|
||||
now: new Date(Date.now() - 26 * 3600 * 1000) // 26 小时前
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'a.json'), JSON.stringify({ payload: old }));
|
||||
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.equal(r.sent, 1);
|
||||
assert.equal(r.failed, 0);
|
||||
const sent = JSON.parse(gw.got[0].body);
|
||||
assert.match(sent.subject, /^\[补投\] /);
|
||||
assert.match(sent.body, /补投/);
|
||||
assert.match(sent.body, /不代表现在仍在故障/);
|
||||
assert.ok(!existsSync(join(root, 'dsh', 'a.json')), '投递成功要删掉 spool 文件');
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('新鲜的报告不加「补投」标记', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const fresh = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service'
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'b.json'), JSON.stringify({ payload: fresh }));
|
||||
await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.ok(!JSON.parse(gw.got[0].body).subject.startsWith('[补投]'));
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('★ 补投失败必须带回原因（不能只报数）', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway(() => 403);
|
||||
try {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'dsh', AGENTMAIL_AGENT_KEY: 'k', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'dsh.service'
|
||||
});
|
||||
mkdirSync(join(root, 'dsh'), { recursive: true });
|
||||
writeFileSync(join(root, 'dsh', 'c.json'), JSON.stringify({ payload: p }));
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_KEY: 'k', AGENTMAIL_AGENT_NAME: 'dsh' },
|
||||
'dsh.service'
|
||||
);
|
||||
assert.equal(r.sent, 0);
|
||||
assert.equal(r.failed, 1);
|
||||
assert.equal(r.errors.length, 1);
|
||||
assert.match(r.errors[0].reason, /403/, '原因里要能看到 HTTP 状态码');
|
||||
assert.ok(existsSync(join(root, 'dsh', 'c.json')), '失败要保留文件以便下次重试');
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('★ 只有 secret 时用 X-Agent-Name/Secret 投递（zcode 的唯一出路）', async () => {
|
||||
const root = tmp();
|
||||
const gw = await fakeGateway();
|
||||
try {
|
||||
const p = buildFailurePayload({
|
||||
env: { AGENTMAIL_AGENT_NAME: 'zcode', AGENTMAIL_AGENT_SECRET: 'sec-1', SERVICE_RESULT: 'exit-code' },
|
||||
serviceName: 'zcode.service'
|
||||
});
|
||||
mkdirSync(join(root, 'zcode'), { recursive: true });
|
||||
writeFileSync(join(root, 'zcode', 'd.json'), JSON.stringify({ payload: p }));
|
||||
const envDir = tmp();
|
||||
writeFileSync(join(envDir, 'zcode.env'), 'AGENTMAIL_AGENT_SECRET=sec-1\n');
|
||||
const r = await flushSpool(
|
||||
{ AGENTMAIL_FAILURE_SPOOL: root, AGENTMAIL_GATEWAY_URL: gw.url, AGENTMAIL_AGENT_SECRET: '', AGENTMAIL_AGENT_KEY: '', AGENTMAIL_ENV_DIR: envDir },
|
||||
'zcode.service'
|
||||
);
|
||||
assert.equal(r.sent, 1, `应投递成功：${JSON.stringify(r.errors)}`);
|
||||
// ★ 端点是严格解码的：带我们自己的元数据会被 400 拒掉（实测踩过）
|
||||
assert.equal(JSON.parse(gw.got[0].body).created_at_ms, undefined);
|
||||
const h = gw.got[0].headers;
|
||||
assert.equal(h['x-agent-name'], 'zcode');
|
||||
assert.equal(h['x-agent-secret'], 'sec-1');
|
||||
assert.equal(h.authorization, undefined);
|
||||
} finally {
|
||||
await gw.close();
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user