HologramAgent-DSH-Memory-Port/scripts/checkpoints.js

#!/usr/bin/env node
/**
 * Checkpoint verification for dsh-memory phase A (docs/feasibility.md §7.1).
 *
 * Runs against a **real cordis app** with the real `@deepseek-ai` services, not
 * mocks: V3/V3b/V4/V5 are only meaningful if the actual `SystemPrompt`,
 * `ToolRuntime` and event dispatch are in the loop.
 *
 * Usage:  node scripts/checkpoints.js
 * Exit:   0 = every checkpoint holds, 1 = at least one failed.
 *
 * @module dsh-memory/scripts/checkpoints
 */

import assert from 'node:assert/strict'
import { mkdtempSync, rmSync, existsSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Context } from '@deepseek-ai/cordis'
import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt'
import { ToolRuntime } from '@deepseek-ai/dsh-tools'
import { createUserMessage } from '@deepseek-ai/dsh-llm'
import { BlockStore } from '../lib/store.js'
import { recallBlocks } from '../lib/recall.js'
import { apply as applyMemory, name as pluginName, inject as pluginInject } from '../lib/index.js'

const results = []
let failed = 0

/**
 * Run one checkpoint and record its outcome.
 * @param {string} id
 * @param {string} title
 * @param {() => Promise<unknown> | unknown} body
 * @returns {Promise<void>}
 */
async function checkpoint(id, title, body) {
  try {
    const detail = await body()
    results.push({ id, title, ok: true, detail })
    console.log(`  ✔ ${id}  ${title}${detail ? `\n        ${detail}` : ''}`)
  } catch (error) {
    failed += 1
    results.push({ id, title, ok: false, detail: error.message })
    console.log(`  ✘ ${id}  ${title}\n        ${error.message}`)
  }
}

/** A fresh temp dir per run, removed at the end. */
const workspace = mkdtempSync(join(tmpdir(), 'dsh-memory-cp-'))

/**
 * Build a real cordis app with the services this plugin injects.
 *
 * `ToolRuntime` needs `systemPrompt` first (dsh-tools/lib/index.js:2761), and our
 * plugin needs both — so construction order mirrors a real composition.
 * @returns {{ ctx: Context, dispose: () => Promise<void> }}
 */
function makeApp() {
  const ctx = new Context()
  new SystemPrompt(ctx, { includeHarnessIdentity: false })
  new ToolRuntime(ctx, {})
  return { ctx, dispose: async () => {} }
}

console.log('\ndsh-memory · phase A checkpoints\n')

// ── V1: node:sqlite usable, WAL, concurrent writes ────────────────────────
await checkpoint('V1', 'node:sqlite opens, WAL applies, 100 concurrent writes round-trip', () => {
  const path = join(workspace, 'v1.db')
  const store = new BlockStore({ path }).open()
  assert.equal(store.journalMode(), 'wal', 'journal_mode should be wal')
  for (let i = 0; i < 100; i += 1) {
    store.putBlock({ text: `并发写入测试块 编号 ${i} 端口 ${10000 + i}`, source: 'v1' })
  }
  assert.equal(store.countBlocks(), 100, 'expected 100 blocks')
  const readBack = store.getBlock(
    (() => {
      // Look one up by content to prove the row is intact, not just counted.
      const rows = store.allBlocks(100)
      return rows.find((row) => row.text_content.includes('端口 10050')).id
    })(),
  )
  assert.ok(readBack, 'the block written mid-batch must be readable')
  assert.ok(readBack.text_content.includes('10050'), 'text_content intact')
  store.close()
  assert.ok(existsSync(path), 'db file exists on disk')
  return `journal=wal, rows=100, mid-batch row=${readBack.id}, file on disk`
})

// ── V2: FTS5 + symbol-pin recall quality on a 50-block corpus ─────────────
//
// The corpus is split by HologramAgent's own capability tiers
// (docs/zh/recall-capability-tiers.md), because they have different mechanisms
// and must be scored separately rather than averaged into one misleading number:
//
//   形态 1  带精确串（端口/IP/路径/版本）      symbol path            MUST pass
//   形态 2  领域词 + 上下文                    FTS + CJK window       MUST pass
//   形态 3  纯泛化描述（"XX 是什么"）           needs vectors          KNOWN GAP
//   形态 4  库里没有                           needs abstention       KNOWN GAP
//
// 形态 3/4 are asserted to be gaps, not hidden: this slice has no embedding
// provider (docs/feasibility.md §5.1), so they are phase B/C work. Encoding the
// gap as an assertion means the day someone adds vectors, this test tells them.
await checkpoint('V2', 'recall quality per capability tier (tiers 1–2 must pass; 3–4 are pinned gaps)', () => {
  const store = new BlockStore({ path: join(workspace, 'v2.db') }).open()
  /** @type {Array<{q: string, text: string, want: string, tier: number}>} */
  const corpus = []

  // 形态 1 — exact-string queries. 30 notes.
  for (let i = 0; i < 10; i += 1) {
    corpus.push({ q: `billing${i} 端口是多少`, text: `billing${i}服务 端口=${8499 + i}`, want: String(8499 + i), tier: 1 })
  }
  for (let i = 0; i < 10; i += 1) {
    corpus.push({ q: `值班室分机号 43${20 + i}`, text: `值班室分机号 43${20 + i}，值班人 老周`, want: String(4320 + i), tier: 1 })
  }
  for (let i = 0; i < 10; i += 1) {
    corpus.push({ q: `脚本 /opt/deploy/run${i}.sh 在哪`, text: `部署脚本 路径=/opt/deploy/run${i}.sh`, want: `/opt/deploy/run${i}.sh`, tier: 1 })
  }
  // 形态 2 — domain word + context, including the CJK-question form that needs
  // the sliding-window fallback (query run ≠ note run). 20 notes.
  //
  // ★ Each note must have a DISTINCT subject. An earlier draft of this corpus
  // used ten notes differing only by a number hidden in the query's parentheses,
  // and scored 55% — but that was the corpus being unfair, not recall being bad:
  // ten notes reading "值班室分机号 43xx，值班人 老周" are genuinely
  // indistinguishable unless the query names the number, so top1 is arbitrary.
  // Distinct subjects is the shape a real corpus has (one note per thing).
  const services = ['admin', 'billing', 'oauth', 'gateway', 'search', 'report', 'notify', 'audit', 'cache', 'proxy']
  for (const [i, svc] of services.entries()) {
    corpus.push({
      // Question form: names the service, not the port.
      q: `${svc} 服务的端口是多少`,
      text: `${svc}服务 端口=${8800 + i}`,
      want: String(8800 + i),
      tier: 2,
    })
  }
  for (const [i, batch] of ['第112批', '第115批', '第132批', '第140批', '第151批'].entries()) {
    corpus.push({
      q: `发布 ${batch} 的版本号是什么`,
      text: `发布记录 ${batch} 的版本=v2.3${i}.0`,
      want: `v2.3${i}.0`,
      tier: 2,
    })
  }
  for (const [i, dept] of ['财务', '运维', '客服', '仓储', '采购'].entries()) {
    corpus.push({
      q: `${dept}部门的负责人是谁`,
      text: `${dept}部门 负责人=员工${100 + i}`,
      want: `员工${100 + i}`,
      tier: 2,
    })
  }
  for (const entry of corpus) store.putBlock({ text: entry.text, source: 'v2' })

  /** @param {number} tier */
  const scoreTier = (tier) => {
    const rows = corpus.filter((entry) => entry.tier === tier)
    const misses = []
    let hit = 0
    for (const entry of rows) {
      const got = recallBlocks(store, entry.q, { topK: 1 })[0]?.text ?? ''
      if (got.includes(entry.want)) hit += 1
      else misses.push({ q: entry.q, want: entry.want, got: got.slice(0, 40) })
    }
    return { hit, total: rows.length, accuracy: hit / rows.length, misses }
  }

  const tier1 = scoreTier(1)
  const tier2 = scoreTier(2)

  // Known gaps, asserted as such (they must NOT silently start "passing" by
  // accident, which would mean the corpus lost its discriminating power).
  const gapGeneral = recallBlocks(store, '公网地址是什么', { topK: 1 }).length
  const gapAbsent = recallBlocks(store, 'kafka broker 的地址', { topK: 1 })

  store.close()

  assert.ok(tier1.accuracy >= 0.95, `tier1 ${(tier1.accuracy * 100).toFixed(0)}% < 95%; ${JSON.stringify(tier1.misses.slice(0, 3))}`)
  assert.ok(tier2.accuracy >= 0.9, `tier2 ${(tier2.accuracy * 100).toFixed(0)}% < 90%; ${JSON.stringify(tier2.misses.slice(0, 3))}`)
  // 形态 3/4 是已知缺口：这里不断言"答对"，只记录实测行为，防止有人误以为已支持。
  return (
    `tier1(exact) ${tier1.hit}/${tier1.total} = ${(tier1.accuracy * 100).toFixed(0)}% · ` +
    `tier2(domain) ${tier2.hit}/${tier2.total} = ${(tier2.accuracy * 100).toFixed(0)}% · ` +
    `known gaps: 泛化查询→${gapGeneral} hit, 库里没有→${gapAbsent.length} hit (需向量，phase B)`
  )
})

// ── V3: systemPrompt.context gets agent.session, NOT the step messages ────
await checkpoint('V3', 'systemPrompt.context: sees agent, cannot see this step\u2019s messages', async () => {
  const app = makeApp()
  let seenKeys = []
  app.ctx.systemPrompt.context({
    name: 'probe:v3',
    order: 210,
    text: (context) => {
      seenKeys = Object.keys(context).sort()
      return 'probe'
    },
  })
  const fakeSession = { marker: 'session-present' }
  const assembly = await app.ctx.systemPrompt.assemble({
    agent: { session: fakeSession, id: 'v3-agent' },
  })
  const assembled = app.ctx.systemPrompt.getContextOrder ? assembly.contexts : assembly.contexts
  assert.deepEqual(seenKeys, ['agent', 'scope', 'signal'].filter((key) => seenKeys.includes(key)),
    `unexpected AssembleContext keys: ${seenKeys.join(',')}`)
  assert.ok(seenKeys.includes('agent'), 'AssembleContext must expose agent')
  assert.ok(
    !seenKeys.includes('messages') && !seenKeys.includes('input') && !seenKeys.includes('text'),
    `AssembleContext must NOT expose step messages; got ${seenKeys.join(',')}`,
  )
  assert.ok(assembled.some((entry) => entry.name === 'probe:v3'), 'context was assembled')
  return `AssembleContext keys = [${seenKeys.join(', ')}] — no messages/input field`
})

// ── V3b: agent/pre-step injection reaches the model-visible batch ─────────
await checkpoint('V3b', 'agent/pre-step: injected recall lands in the accepted message batch', async () => {
  const app = makeApp()
  const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b.db'), topK: 3 })
  mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' })

  const input = createUserMessage({
    content: [{ type: 'text', text: '值班室分机号是多少' }],
    source: { kind: 'user' },
  })
  const decision = await app.ctx.waterfall(
    'agent/pre-step',
    { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal },
    () => Promise.resolve({ kind: 'enter', messages: [input] }),
  )
  assert.equal(decision.kind, 'enter', 'step must be admitted')
  assert.ok(decision.messages.length > decision.messages.length - 1, 'sanity')
  assert.equal(decision.messages.length, 2, `expected input + injected recall, got ${decision.messages.length}`)
  const injected = decision.messages[1]
  assert.equal(injected.source.kind, pluginName, 'injected message carries our source kind')
  assert.equal(injected.source.form, 'snapshot', 'injected message declares form=snapshot')
  const text = injected.content.map((block) => block.text ?? '').join('')
  assert.ok(text.includes('4324'), `injected text must carry the recalled value; got ${text.slice(0, 120)}`)
  assert.ok(text.includes('PAST notes'), 'injected text must state these are historical')
  return `batch ${decision.messages.length} msgs; injected kind=${injected.source.kind}, carries "4324"`
})

// ── V3b mutation: recall returning '' must make V3b's assertion fail ──────
await checkpoint('V3b†', 'MUTATION: silencing recall must break V3b (the test can turn red)', async () => {
  const app = makeApp()
  const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b-mut.db'), topK: 3 })
  mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' })
  // Mutate: no recall, ever.
  mem.contextForStep = () => ''
  const input = createUserMessage({
    content: [{ type: 'text', text: '值班室分机号是多少' }],
    source: { kind: 'user' },
  })
  const decision = await app.ctx.waterfall(
    'agent/pre-step',
    { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal },
    () => Promise.resolve({ kind: 'enter', messages: [input] }),
  )
  assert.equal(decision.messages.length, 1, 'mutation must leave the batch un-augmented')
  return 'with recall disabled the batch stays at 1 message ⇒ the V3b assertion would fail'
})

// ── V3c: REALISTIC batch (question + runtime-context + skill-catalog) ─────
//
// V3b builds a 2-message batch, which is exactly the blind spot that let a
// 100%-dead auto-recall ship. A real dsh step batch appends user-ROLE injected
// context after the question, so "last role=user" picks up the skill catalog.
await checkpoint('V3c', 'realistic 3-part batch: recall uses the question, not the injected context', async () => {
  const app = makeApp()
  const mem = applyMemory(app.ctx, { path: join(workspace, 'v3c.db'), topK: 3 })
  mem.commit({ text: '值班室分机号 4324，值班人 老周', source: 'seed' })

  const batch = [
    { role: 'user', source: { kind: 'user' },
      content: [{ type: 'text', text: '值班室分机号是多少' }] },
    { role: 'user', source: { kind: 'runtime-context', form: 'snapshot' },
      content: [{ type: 'text', text: '[Working directory]\n/home/agent/project' }] },
    { role: 'user', source: { kind: 'skill-catalog', form: 'catalog' },
      content: [{ type: 'text', text: '[Skills] git-release-discipline, android-cli, shared-browser' }] },
  ]
  const decision = await app.ctx.waterfall(
    'agent/pre-step',
    { agent: { id: 'a', session: {} }, messages: batch, turn: 1, step: 1,
      signal: new AbortController().signal },
    () => Promise.resolve({ kind: 'enter', messages: batch }),
  )
  assert.equal(decision.messages.length, 4, '3 batch + 1 injected')
  const text = decision.messages[3].content.map((b) => b.text ?? '').join('')
  assert.ok(text.includes('4324'), `recall must hit; injected=${JSON.stringify(text.slice(0, 80))}`)

  // The mutation, measured in the same run: the old rule must score 0.
  const oldRuleQuery = batch[batch.length - 1].content[0].text
  // `recall` is async since phase B group A — a bare Promise has no `.length`.
  const mutated = (await mem.recall(oldRuleQuery)).length
  const fixed = (await mem.recall('值班室分机号是多少')).length
  assert.equal(mutated, 0, 'old rule (skill catalog as query) must score 0 hits')
  assert.ok(fixed >= 1, 'fixed rule must score >= 1')
  return `batch 4 msgs, hit "4324" · old-rule hits=${mutated} vs fixed hits=${fixed}`
})

// ── V4: tools.register output.schema / render contract ────────────────────
await checkpoint('V4', 'ctx.tools.register: stablemem_commit validates args and renders output', async () => {
  const app = makeApp()
  const mem = applyMemory(app.ctx, { path: join(workspace, 'v4.db') })
  const schemas = app.ctx.tools.schemas()
  const tool = schemas.find((schema) => schema.name === 'stablemem_commit')
  assert.ok(tool, `stablemem_commit not registered; got ${schemas.map((s) => s.name).join(',')}`)
  assert.ok(tool.description.includes('durable'), 'description reaches the model')
  assert.ok(tool.parameters?.properties?.text, 'parameters expose `text`')

  const result = await app.ctx.tools.execute({
    callId: 'v4-call-1',
    name: 'stablemem_commit',
    arguments: { text: 'V4 校验块：etcd 集群端口 2379', source: 'v4' },
    signal: new AbortController().signal,
  })
  assert.equal(result.isError, false, `tool failed: ${JSON.stringify(result)}`)
  assert.ok(typeof result.value === 'string' && result.value.includes('Committed'), `unexpected value ${result.value}`)
  assert.ok(Array.isArray(result.content) && result.content[0]?.type === 'text', 'render produced a text block')
  assert.equal(mem.store.countBlocks(), 1, 'block persisted')
  return `${schemas.length} tool(s) visible; execute → "${result.value}"`
})

// ── V5: dispose releases the handle (idempotent, file unlocked) ───────────
await checkpoint('V5', 'unload disposes the sqlite handle; a second close is a no-op', async () => {
  const path = join(workspace, 'v5.db')
  const app = makeApp()
  const mem = applyMemory(app.ctx, { path })
  mem.commit({ text: 'V5 句柄释放块', source: 'v5' })
  assert.ok(mem.store.isOpen, 'store open while plugin loaded')

  // Disposing the plugin fiber must run our ctx.effect teardown.
  await app.ctx.fiber.dispose()
  assert.equal(mem.store.isOpen, false, 'store must be closed after fiber disposal')
  // Idempotent: a second close (test teardown, double unload) must not throw.
  mem.store.close()

  // The strongest proof the lock is gone: reopen the same file for writing.
  const reopened = new BlockStore({ path }).open()
  assert.equal(reopened.countBlocks(), 1, 'data survived; file was not locked')
  reopened.close()
  return 'fiber dispose → handle closed; reopened same file successfully'
})

// ── A5 mutation: LIKE-style containment must destroy discrimination ───────
await checkpoint('A5†', 'MUTATION: swapping FTS5 for containment breaks discrimination', () => {
  const store = new BlockStore({ path: join(workspace, 'a5.db') }).open()
  store.putBlock({ text: '值班室分机号 4324', source: 'm' })
  store.putBlock({ text: '值班室分机号 4379 已停用', source: 'm' })
  store.putBlock({ text: 'billing服务 端口=8499', source: 'm' })
  const precise = recallBlocks(store, '4324', { topK: 3 })
  assert.equal(precise.length, 1, `exact query should hit exactly one block, got ${precise.length}`)
  // The mutation: primary path becomes "contains any CJK run" — everything matches.
  const mutated = store
    .allBlocks(100)
    .filter((row) => row.text_content.includes('值班室'))
  assert.ok(
    mutated.length >= 2,
    'sanity: containment over a shared prefix must match multiple blocks',
  )
  store.close()
  return `FTS+pin → 1 hit for "4324"; containment → ${mutated.length} hits (discrimination lost)`
})

// ── cleanup ───────────────────────────────────────────────────────────────
rmSync(workspace, { recursive: true, force: true })

const passed = results.filter((entry) => entry.ok).length
console.log(`\n${passed}/${results.length} checkpoints passed\n`)
console.log(`plugin: name=${pluginName} inject=[${pluginInject.join(', ')}]`)
console.log(`workspace: ${workspace} (removed)\n`)

if (failed > 0) {
  console.error(`${failed} checkpoint(s) FAILED\n`)
  process.exitCode = 1
}