#!/usr/bin/env node /** * systemd 服务异常退出邮件上报器。 * * 两种调用: * service-failure-notify.mjs --report --service pi-mail-bridge.service * service-failure-notify.mjs --flush --service pi-mail-bridge.service * * `--report` 供 ExecStopPost 使用。systemd 会把 SERVICE_RESULT / EXIT_CODE / * EXIT_STATUS / INVOCATION_ID 注入命令环境;正常 stop/restart 不上报,只有 * exit-code、signal、oom-kill、timeout、watchdog 等异常结果才发信。 * * Gateway 暂时不可达时,报告只把**不含密钥**的邮件 payload 落到本地 spool; * 下次服务启动由 `--flush` 补发。relay_key 取 systemd invocation id,立即发送与 * 补发即使竞态也只会入库一封。 */ import { createHash, randomUUID } from 'node:crypto'; import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; const DEFAULT_GATEWAY = 'http://127.0.0.1:8180'; const DEFAULT_RECIPIENT = 'jianf@'; const DEFAULT_SPOOL = '/var/lib/agentmail/failure-spool'; const MAX_SPOOL_FILES = 100; const REQUEST_TIMEOUT_MS = 5_000; function argValue(argv, name) { const i = argv.indexOf(name); return i >= 0 && i + 1 < argv.length ? argv[i + 1] : ''; } export function isAbnormalExit(env = process.env) { const result = String(env.SERVICE_RESULT || '').trim(); if (result) return result !== 'success'; const code = String(env.EXIT_CODE || '').trim(); const status = String(env.EXIT_STATUS || '').trim(); if (!code && !status) return false; return !(code === 'exited' && (status === '' || status === '0')); } function cleanServiceName(value, agentName) { const raw = String(value || '').trim(); if (raw) return basename(raw).replace(/[^A-Za-z0-9_.@-]/g, '-').slice(0, 96); return `${String(agentName || 'agent').replace(/[^A-Za-z0-9_.@-]/g, '-')}-bridge`; } function redact(value, env) { let out = String(value || ''); for (const secret of [env.AGENTMAIL_AGENT_KEY, env.AGENTMAIL_AGENT_SECRET]) { if (secret) out = out.split(String(secret)).join('[REDACTED]'); } return out .replace(/Bearer\s+[A-Za-z0-9._~+\/-]+/gi, 'Bearer [REDACTED]') .replace(/\b(?:ak_[A-Za-z0-9_-]+|sk-[A-Za-z0-9_-]{12,})\b/g, '[REDACTED]') .slice(0, 1_500); } function relayKeyFor(env, serviceName) { const invocation = String(env.INVOCATION_ID || '').trim(); if (invocation) return `service-failure:${invocation}`; const seed = `${serviceName}:${Date.now()}:${randomUUID()}`; return `service-failure:${createHash('sha256').update(seed).digest('hex')}`; } export function buildFailurePayload({ env = process.env, serviceName = '', reason = '', error = null, now = new Date(), } = {}) { const agentName = String(env.AGENTMAIL_AGENT_NAME || 'unknown-agent').trim(); const service = cleanServiceName(serviceName || env.AGENTMAIL_SERVICE_NAME, agentName); const result = String(env.SERVICE_RESULT || reason || 'unexpected-exit'); const exitCode = String(env.EXIT_CODE || 'unknown'); const exitStatus = String(env.EXIT_STATUS || 'unknown'); const detail = redact(error?.stack || error?.message || error || reason, env); const lines = [ `**${agentName}** 的宿主服务 **${service}** 异常终止。`, '', `- 时间:${now.toISOString()}`, `- systemd 结果:${result}`, `- 退出类型:${exitCode}`, `- 退出状态:${exitStatus}`, `- invocation:${String(env.INVOCATION_ID || 'unknown')}`, ]; if (detail) lines.push('', '**异常详情(已脱敏)**:', '```text', detail, '```'); lines.push( '', 'systemd 已按指数退避策略自动重启该服务;如果邮件连续出现,请检查:', `\`journalctl -u ${service} -n 100 --no-pager\``, ); return { to: String(env.AGENTMAIL_FAILURE_RECIPIENT || DEFAULT_RECIPIENT), subject: `[${agentName}] 桥服务异常终止`, body: lines.join('\n'), relay: 'summary', relay_key: relayKeyFor(env, service), }; } async function postPayload(payload, env = process.env) { const key = String(env.AGENTMAIL_AGENT_KEY || '').trim(); if (!key) throw new Error('AGENTMAIL_AGENT_KEY 未配置'); const base = String(env.AGENTMAIL_GATEWAY_URL || DEFAULT_GATEWAY).replace(/\/+$/, ''); const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); try { const res = await fetch(`${base}/api/v1/mail/send`, { method: 'POST', headers: { Authorization: `Bearer ${key}`, 'X-Agent-Name': String(env.AGENTMAIL_AGENT_NAME || ''), 'Content-Type': 'application/json', }, body: JSON.stringify(payload), signal: controller.signal, }); const responseBody = await res.text(); if (!res.ok) throw new Error(`Gateway HTTP ${res.status}: ${responseBody.slice(0, 300)}`); return responseBody; } finally { clearTimeout(timer); } } function safeAgentName(env) { return String(env.AGENTMAIL_AGENT_NAME || 'unknown-agent') .replace(/[^A-Za-z0-9_.@-]/g, '-') .slice(0, 96); } function spoolDirFor(env) { return join(String(env.AGENTMAIL_FAILURE_SPOOL || DEFAULT_SPOOL), safeAgentName(env)); } function safeSpoolName(payload) { return `${createHash('sha256').update(String(payload.relay_key)).digest('hex')}.json`; } async function pruneSpool(spoolDir) { const names = (await readdir(spoolDir).catch(() => [])) .filter((name) => name.endsWith('.json')); if (names.length <= MAX_SPOOL_FILES) return; const entries = await Promise.all(names.map(async (name) => ({ name, mtime: (await stat(join(spoolDir, name))).mtimeMs, }))); entries.sort((a, b) => a.mtime - b.mtime); for (const entry of entries.slice(0, entries.length - MAX_SPOOL_FILES)) { await rm(join(spoolDir, entry.name), { force: true }); } } export async function spoolPayload(payload, env = process.env) { // 每个 Agent 独立目录:否则 pi 启动时可能拿自己的密钥去补发 dsh 的报告, // 发件身份与正文主体会错位,relay_key 的幂等范围也变了。 const spoolDir = spoolDirFor(env); await mkdir(spoolDir, { recursive: true, mode: 0o700 }); const target = join(spoolDir, safeSpoolName(payload)); const temp = `${target}.${process.pid}.${randomUUID()}.tmp`; await writeFile(temp, `${JSON.stringify({ payload }, null, 2)}\n`, { mode: 0o600 }); await rename(temp, target); await pruneSpool(spoolDir); return target; } export async function notifyCrash(reason, error, options = {}) { const env = options.env || process.env; const payload = buildFailurePayload({ env, serviceName: options.serviceName, reason, error, now: options.now, }); try { await postPayload(payload, env); return { sent: true, spooled: false, payload }; } catch (sendError) { if (options.spool === false) throw sendError; const path = await spoolPayload(payload, env); return { sent: false, spooled: true, path, payload, error: sendError }; } } export async function flushSpool(env = process.env) { const spoolDir = spoolDirFor(env); const names = (await readdir(spoolDir).catch(() => [])) .filter((name) => name.endsWith('.json')) .sort(); let sent = 0; let failed = 0; for (const name of names) { const path = join(spoolDir, name); try { const record = JSON.parse(await readFile(path, 'utf8')); await postPayload(record.payload, env); await rm(path, { force: true }); sent++; } catch { failed++; } } return { total: names.length, sent, failed }; } async function main(argv = process.argv.slice(2), env = process.env) { const serviceName = argValue(argv, '--service'); if (argv.includes('--flush')) { const result = await flushSpool(env); if (result.sent || result.failed) { console.error(`[agentmail-failure-notify] spool: sent=${result.sent} failed=${result.failed}`); } return; } if (!argv.includes('--report')) throw new Error('需要 --report 或 --flush'); if (!isAbnormalExit(env)) return; if (argv.includes('--dry-run')) { console.log(JSON.stringify(buildFailurePayload({ env, serviceName }), null, 2)); return; } const result = await notifyCrash(env.SERVICE_RESULT || 'unexpected-exit', null, { env, serviceName, }); console.error(result.sent ? `[agentmail-failure-notify] ${serviceName} 异常已上报` : `[agentmail-failure-notify] Gateway 不可达,报告已暂存 ${result.path}`); } const isCLI = process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url); if (isCLI) { main().catch((error) => { console.error(`[agentmail-failure-notify] ${error?.message || error}`); process.exitCode = 1; }); }