#!/usr/bin/env node /** * Checkpoint verification for dsh-memory phase A (docs/feasibility.md §7.1). * * Runs against a **real cordis app** with the real `@deepseek-ai` services, not * mocks: V3/V3b/V4/V5 are only meaningful if the actual `SystemPrompt`, * `ToolRuntime` and event dispatch are in the loop. * * Usage: node scripts/checkpoints.js * Exit: 0 = every checkpoint holds, 1 = at least one failed. * * @module dsh-memory/scripts/checkpoints */ import assert from 'node:assert/strict' import { mkdtempSync, rmSync, existsSync } from 'node:fs' import { DatabaseSync } from 'node:sqlite' import { tmpdir } from 'node:os' import { join } from 'node:path' import { Context } from '@deepseek-ai/cordis' import { SystemPrompt } from '@deepseek-ai/dsh-system-prompt' import { ToolRuntime } from '@deepseek-ai/dsh-tools' import { createUserMessage } from '@deepseek-ai/dsh-llm' import { BlockStore } from '../lib/store.js' import { recallBlocks } from '../lib/recall.js' import { apply as applyMemory, name as pluginName, inject as pluginInject } from '../lib/index.js' const results = [] let failed = 0 /** * Run one checkpoint and record its outcome. * @param {string} id * @param {string} title * @param {() => Promise | unknown} body * @returns {Promise} */ async function checkpoint(id, title, body) { try { const detail = await body() results.push({ id, title, ok: true, detail }) console.log(` ✔ ${id} ${title}${detail ? `\n ${detail}` : ''}`) } catch (error) { failed += 1 results.push({ id, title, ok: false, detail: error.message }) console.log(` ✘ ${id} ${title}\n ${error.message}`) } } /** A fresh temp dir per run, removed at the end. */ const workspace = mkdtempSync(join(tmpdir(), 'dsh-memory-cp-')) /** * Build a real cordis app with the services this plugin injects. * * `ToolRuntime` needs `systemPrompt` first (dsh-tools/lib/index.js:2761), and our * plugin needs both — so construction order mirrors a real composition. * @returns {{ ctx: Context, dispose: () => Promise }} */ function makeApp() { const ctx = new Context() new SystemPrompt(ctx, { includeHarnessIdentity: false }) new ToolRuntime(ctx, {}) return { ctx, dispose: async () => {} } } console.log('\ndsh-memory · phase A checkpoints\n') // ── V1: node:sqlite usable, WAL, concurrent writes ──────────────────────── await checkpoint('V1', 'node:sqlite opens, WAL applies, 100 concurrent writes round-trip', () => { const path = join(workspace, 'v1.db') const store = new BlockStore({ path }).open() assert.equal(store.journalMode(), 'wal', 'journal_mode should be wal') for (let i = 0; i < 100; i += 1) { store.putBlock({ text: `并发写入测试块 编号 ${i} 端口 ${10000 + i}`, source: 'v1' }) } assert.equal(store.countBlocks(), 100, 'expected 100 blocks') const readBack = store.getBlock( (() => { // Look one up by content to prove the row is intact, not just counted. const rows = store.allBlocks(100) return rows.find((row) => row.text_content.includes('端口 10050')).id })(), ) assert.ok(readBack, 'the block written mid-batch must be readable') assert.ok(readBack.text_content.includes('10050'), 'text_content intact') store.close() assert.ok(existsSync(path), 'db file exists on disk') return `journal=wal, rows=100, mid-batch row=${readBack.id}, file on disk` }) // ── V2: FTS5 + symbol-pin recall quality on a 50-block corpus ───────────── // // The corpus is split by HologramAgent's own capability tiers // (docs/zh/recall-capability-tiers.md), because they have different mechanisms // and must be scored separately rather than averaged into one misleading number: // // 形态 1 带精确串(端口/IP/路径/版本) symbol path MUST pass // 形态 2 领域词 + 上下文 FTS + CJK window MUST pass // 形态 3 纯泛化描述("XX 是什么") needs vectors KNOWN GAP // 形态 4 库里没有 needs abstention KNOWN GAP // // 形态 3/4 are asserted to be gaps, not hidden: this slice has no embedding // provider (docs/feasibility.md §5.1), so they are phase B/C work. Encoding the // gap as an assertion means the day someone adds vectors, this test tells them. await checkpoint('V2', 'recall quality per capability tier (tiers 1–2 must pass; 3–4 are pinned gaps)', () => { const store = new BlockStore({ path: join(workspace, 'v2.db') }).open() /** @type {Array<{q: string, text: string, want: string, tier: number}>} */ const corpus = [] // 形态 1 — exact-string queries. 30 notes. for (let i = 0; i < 10; i += 1) { corpus.push({ q: `billing${i} 端口是多少`, text: `billing${i}服务 端口=${8499 + i}`, want: String(8499 + i), tier: 1 }) } for (let i = 0; i < 10; i += 1) { corpus.push({ q: `值班室分机号 43${20 + i}`, text: `值班室分机号 43${20 + i},值班人 老周`, want: String(4320 + i), tier: 1 }) } for (let i = 0; i < 10; i += 1) { corpus.push({ q: `脚本 /opt/deploy/run${i}.sh 在哪`, text: `部署脚本 路径=/opt/deploy/run${i}.sh`, want: `/opt/deploy/run${i}.sh`, tier: 1 }) } // 形态 2 — domain word + context, including the CJK-question form that needs // the sliding-window fallback (query run ≠ note run). 20 notes. // // ★ Each note must have a DISTINCT subject. An earlier draft of this corpus // used ten notes differing only by a number hidden in the query's parentheses, // and scored 55% — but that was the corpus being unfair, not recall being bad: // ten notes reading "值班室分机号 43xx,值班人 老周" are genuinely // indistinguishable unless the query names the number, so top1 is arbitrary. // Distinct subjects is the shape a real corpus has (one note per thing). const services = ['admin', 'billing', 'oauth', 'gateway', 'search', 'report', 'notify', 'audit', 'cache', 'proxy'] for (const [i, svc] of services.entries()) { corpus.push({ // Question form: names the service, not the port. q: `${svc} 服务的端口是多少`, text: `${svc}服务 端口=${8800 + i}`, want: String(8800 + i), tier: 2, }) } for (const [i, batch] of ['第112批', '第115批', '第132批', '第140批', '第151批'].entries()) { corpus.push({ q: `发布 ${batch} 的版本号是什么`, text: `发布记录 ${batch} 的版本=v2.3${i}.0`, want: `v2.3${i}.0`, tier: 2, }) } for (const [i, dept] of ['财务', '运维', '客服', '仓储', '采购'].entries()) { corpus.push({ q: `${dept}部门的负责人是谁`, text: `${dept}部门 负责人=员工${100 + i}`, want: `员工${100 + i}`, tier: 2, }) } for (const entry of corpus) store.putBlock({ text: entry.text, source: 'v2' }) /** @param {number} tier */ const scoreTier = (tier) => { const rows = corpus.filter((entry) => entry.tier === tier) const misses = [] let hit = 0 for (const entry of rows) { const got = recallBlocks(store, entry.q, { topK: 1 })[0]?.text ?? '' if (got.includes(entry.want)) hit += 1 else misses.push({ q: entry.q, want: entry.want, got: got.slice(0, 40) }) } return { hit, total: rows.length, accuracy: hit / rows.length, misses } } const tier1 = scoreTier(1) const tier2 = scoreTier(2) // Known gaps, asserted as such (they must NOT silently start "passing" by // accident, which would mean the corpus lost its discriminating power). const gapGeneral = recallBlocks(store, '公网地址是什么', { topK: 1 }).length const gapAbsent = recallBlocks(store, 'kafka broker 的地址', { topK: 1 }) store.close() assert.ok(tier1.accuracy >= 0.95, `tier1 ${(tier1.accuracy * 100).toFixed(0)}% < 95%; ${JSON.stringify(tier1.misses.slice(0, 3))}`) assert.ok(tier2.accuracy >= 0.9, `tier2 ${(tier2.accuracy * 100).toFixed(0)}% < 90%; ${JSON.stringify(tier2.misses.slice(0, 3))}`) // 形态 3/4 是已知缺口:这里不断言"答对",只记录实测行为,防止有人误以为已支持。 return ( `tier1(exact) ${tier1.hit}/${tier1.total} = ${(tier1.accuracy * 100).toFixed(0)}% · ` + `tier2(domain) ${tier2.hit}/${tier2.total} = ${(tier2.accuracy * 100).toFixed(0)}% · ` + `known gaps: 泛化查询→${gapGeneral} hit, 库里没有→${gapAbsent.length} hit (需向量,phase B)` ) }) // ── V3: systemPrompt.context gets agent.session, NOT the step messages ──── await checkpoint('V3', 'systemPrompt.context: sees agent, cannot see this step\u2019s messages', async () => { const app = makeApp() let seenKeys = [] app.ctx.systemPrompt.context({ name: 'probe:v3', order: 210, text: (context) => { seenKeys = Object.keys(context).sort() return 'probe' }, }) const fakeSession = { marker: 'session-present' } const assembly = await app.ctx.systemPrompt.assemble({ agent: { session: fakeSession, id: 'v3-agent' }, }) const assembled = app.ctx.systemPrompt.getContextOrder ? assembly.contexts : assembly.contexts assert.deepEqual(seenKeys, ['agent', 'scope', 'signal'].filter((key) => seenKeys.includes(key)), `unexpected AssembleContext keys: ${seenKeys.join(',')}`) assert.ok(seenKeys.includes('agent'), 'AssembleContext must expose agent') assert.ok( !seenKeys.includes('messages') && !seenKeys.includes('input') && !seenKeys.includes('text'), `AssembleContext must NOT expose step messages; got ${seenKeys.join(',')}`, ) assert.ok(assembled.some((entry) => entry.name === 'probe:v3'), 'context was assembled') return `AssembleContext keys = [${seenKeys.join(', ')}] — no messages/input field` }) // ── V3b: agent/pre-step injection reaches the model-visible batch ───────── await checkpoint('V3b', 'agent/pre-step: injected recall lands in the accepted message batch', async () => { const app = makeApp() const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) const input = createUserMessage({ content: [{ type: 'text', text: '值班室分机号是多少' }], source: { kind: 'user' }, }) const decision = await app.ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal }, () => Promise.resolve({ kind: 'enter', messages: [input] }), ) assert.equal(decision.kind, 'enter', 'step must be admitted') assert.ok(decision.messages.length > decision.messages.length - 1, 'sanity') assert.equal(decision.messages.length, 2, `expected input + injected recall, got ${decision.messages.length}`) const injected = decision.messages[1] assert.equal(injected.source.kind, pluginName, 'injected message carries our source kind') assert.equal(injected.source.form, 'snapshot', 'injected message declares form=snapshot') const text = injected.content.map((block) => block.text ?? '').join('') assert.ok(text.includes('4324'), `injected text must carry the recalled value; got ${text.slice(0, 120)}`) assert.ok(text.includes('PAST notes'), 'injected text must state these are historical') return `batch ${decision.messages.length} msgs; injected kind=${injected.source.kind}, carries "4324"` }) // ── V3b mutation: recall returning '' must make V3b's assertion fail ────── await checkpoint('V3b†', 'MUTATION: silencing recall must break V3b (the test can turn red)', async () => { const app = makeApp() const mem = applyMemory(app.ctx, { path: join(workspace, 'v3b-mut.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) // Mutate: no recall, ever. mem.contextForStep = () => '' const input = createUserMessage({ content: [{ type: 'text', text: '值班室分机号是多少' }], source: { kind: 'user' }, }) const decision = await app.ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: [input], turn: 1, step: 1, signal: new AbortController().signal }, () => Promise.resolve({ kind: 'enter', messages: [input] }), ) assert.equal(decision.messages.length, 1, 'mutation must leave the batch un-augmented') return 'with recall disabled the batch stays at 1 message ⇒ the V3b assertion would fail' }) // ── V3c: REALISTIC batch (question + runtime-context + skill-catalog) ───── // // V3b builds a 2-message batch, which is exactly the blind spot that let a // 100%-dead auto-recall ship. A real dsh step batch appends user-ROLE injected // context after the question, so "last role=user" picks up the skill catalog. await checkpoint('V3c', 'realistic 3-part batch: recall uses the question, not the injected context', async () => { const app = makeApp() const mem = applyMemory(app.ctx, { path: join(workspace, 'v3c.db'), topK: 3 }) await mem.commit({ text: '值班室分机号 4324,值班人 老周', source: 'seed' }) const batch = [ { role: 'user', source: { kind: 'user' }, content: [{ type: 'text', text: '值班室分机号是多少' }] }, { role: 'user', source: { kind: 'runtime-context', form: 'snapshot' }, content: [{ type: 'text', text: '[Working directory]\n/home/agent/project' }] }, { role: 'user', source: { kind: 'skill-catalog', form: 'catalog' }, content: [{ type: 'text', text: '[Skills] git-release-discipline, android-cli, shared-browser' }] }, ] const decision = await app.ctx.waterfall( 'agent/pre-step', { agent: { id: 'a', session: {} }, messages: batch, turn: 1, step: 1, signal: new AbortController().signal }, () => Promise.resolve({ kind: 'enter', messages: batch }), ) assert.equal(decision.messages.length, 4, '3 batch + 1 injected') const text = decision.messages[3].content.map((b) => b.text ?? '').join('') assert.ok(text.includes('4324'), `recall must hit; injected=${JSON.stringify(text.slice(0, 80))}`) // The mutation, measured in the same run: the old rule must score 0. const oldRuleQuery = batch[batch.length - 1].content[0].text // `recall` is async since phase B group A — a bare Promise has no `.length`. const mutated = (await mem.recall(oldRuleQuery)).length const fixed = (await mem.recall('值班室分机号是多少')).length assert.equal(mutated, 0, 'old rule (skill catalog as query) must score 0 hits') assert.ok(fixed >= 1, 'fixed rule must score >= 1') return `batch 4 msgs, hit "4324" · old-rule hits=${mutated} vs fixed hits=${fixed}` }) // ── V4: tools.register output.schema / render contract ──────────────────── await checkpoint('V4', 'ctx.tools.register: stablemem_commit validates args and renders output', async () => { const app = makeApp() const mem = applyMemory(app.ctx, { path: join(workspace, 'v4.db') }) const schemas = app.ctx.tools.schemas() const tool = schemas.find((schema) => schema.name === 'stablemem_commit') assert.ok(tool, `stablemem_commit not registered; got ${schemas.map((s) => s.name).join(',')}`) assert.ok(tool.description.includes('durable'), 'description reaches the model') assert.ok(tool.parameters?.properties?.text, 'parameters expose `text`') const result = await app.ctx.tools.execute({ callId: 'v4-call-1', name: 'stablemem_commit', arguments: { text: 'V4 校验块:etcd 集群端口 2379', source: 'v4' }, signal: new AbortController().signal, }) assert.equal(result.isError, false, `tool failed: ${JSON.stringify(result)}`) assert.ok(typeof result.value === 'string' && result.value.includes('Committed'), `unexpected value ${result.value}`) assert.ok(Array.isArray(result.content) && result.content[0]?.type === 'text', 'render produced a text block') assert.equal(mem.store.countBlocks(), 1, 'block persisted') return `${schemas.length} tool(s) visible; execute → "${result.value}"` }) // ── V5: dispose releases the handle (idempotent, file unlocked) ─────────── await checkpoint('V5', 'unload disposes the sqlite handle; a second close is a no-op', async () => { const path = join(workspace, 'v5.db') const app = makeApp() const mem = applyMemory(app.ctx, { path }) await mem.commit({ text: 'V5 句柄释放块', source: 'v5' }) assert.ok(mem.store.isOpen, 'store open while plugin loaded') // Disposing the plugin fiber must run our ctx.effect teardown. await app.ctx.fiber.dispose() assert.equal(mem.store.isOpen, false, 'store must be closed after fiber disposal') // Idempotent: a second close (test teardown, double unload) must not throw. mem.store.close() // The strongest proof the lock is gone: reopen the same file for writing. const reopened = new BlockStore({ path }).open() assert.equal(reopened.countBlocks(), 1, 'data survived; file was not locked') reopened.close() return 'fiber dispose → handle closed; reopened same file successfully' }) // ── A5 mutation: LIKE-style containment must destroy discrimination ─────── await checkpoint('A5†', 'MUTATION: swapping FTS5 for containment breaks discrimination', () => { const store = new BlockStore({ path: join(workspace, 'a5.db') }).open() store.putBlock({ text: '值班室分机号 4324', source: 'm' }) store.putBlock({ text: '值班室分机号 4379 已停用', source: 'm' }) store.putBlock({ text: 'billing服务 端口=8499', source: 'm' }) const precise = recallBlocks(store, '4324', { topK: 3 }) assert.equal(precise.length, 1, `exact query should hit exactly one block, got ${precise.length}`) // The mutation: primary path becomes "contains any CJK run" — everything matches. const mutated = store .allBlocks(100) .filter((row) => row.text_content.includes('值班室')) assert.ok( mutated.length >= 2, 'sanity: containment over a shared prefix must match multiple blocks', ) store.close() return `FTS+pin → 1 hit for "4324"; containment → ${mutated.length} hits (discrimination lost)` }) // ── MIG3: an existing phase-A database survives the phase-B upgrade ─────── // // Lives in `check` as well as `test` on purpose: "can the user's existing // memory.db still be used" is the single most important thing to see at a // glance, and more people read this output than read the test suite. The // failure it guards against is silent at load — the store opens, stamps a // version, accepts writes to blocks, and only throws on the first centroid // write — so a casual smoke test would miss it. await checkpoint('MIG3', 'phase-A database opens, migrates, and stays fully writable', () => { const path = join(workspace, 'phase-a.db') // Build the phase-A shape by hand, including its OLD centroid table. const legacy = new DatabaseSync(path) legacy.exec(`CREATE TABLE memory_blocks ( id TEXT PRIMARY KEY, modality TEXT NOT NULL, text_content TEXT NOT NULL DEFAULT '', source TEXT NOT NULL DEFAULT '', scene TEXT NOT NULL DEFAULT '', semantic_type TEXT NOT NULL DEFAULT '', created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL)`) legacy.exec(`CREATE TABLE graph_centroid ( fingerprint TEXT PRIMARY KEY, dim INTEGER NOT NULL, count INTEGER NOT NULL, sum_json TEXT NOT NULL, updated_at INTEGER NOT NULL)`) legacy.exec(`CREATE VIRTUAL TABLE memory_blocks_fts USING fts5(text_content, block_id UNINDEXED, tokenize='unicode61 remove_diacritics 2')`) legacy.exec(`INSERT INTO memory_blocks (id, modality, text_content, source, created_at, updated_at) VALUES ('blk_legacy', 'text', '值班室分机号 4324', 'phase-a', 1, 1)`) legacy.exec(`INSERT INTO memory_blocks_fts (text_content, block_id) VALUES ('值班室分机号 4324', 'blk_legacy')`) legacy.close() const store = new BlockStore({ path }).open() const version = store.userVersion() const cols = store.columns('memory_blocks') assert.ok(cols.includes('vector') && cols.includes('fingerprint'), `columns: ${cols.join(',')}`) assert.equal(version, 1, `user_version after migration: ${version}`) // The legacy block must survive AND still be findable. assert.ok(store.getBlock('blk_legacy'), 'the phase-A block must survive') assert.ok(store.searchFts('"4324"', 5).length >= 1, 'FTS must still find the legacy block') // Both write paths must work — the block path AND the centroid path. store.putBlock({ text: '迁移之后的新块 8499' }) const centroidCols = store.columns('graph_centroid') store.saveCentroid({ fingerprint: 'fp-mig', dim: 3, vectorCount: 1, mean: [0, 0, 0], isAnomalous: true }) assert.ok(store.loadCentroid('fp-mig'), 'centroid must round-trip after migration') store.close() return `uv=${version}, blocks=${cols.length} cols, centroid=${centroidCols.length} cols, legacy block readable + FTS-matching` }) // ── F7: store-level fingerprint inventory, D4-derived and D7-cheap ──────── await checkpoint('F7', 'store stats groups active and stale fingerprints with DB-derived counts', () => { const path = join(workspace, 'f7.db') const store = new BlockStore({ path }).open() try { store.putBlock({ text: 'F7 active', vector: [1, 0, 0], fingerprint: 'fp-active' }) store.putBlock({ text: 'F7 stale one', vector: [1, 0], fingerprint: 'fp-stale' }) store.putBlock({ text: 'F7 stale two', vector: [0, 1], fingerprint: 'fp-stale' }) store.putBlock({ text: 'F7 no vector' }) store.saveCentroid({ fingerprint: 'fp-active', dim: 3, vectorCount: 1, mean: [0, 0, 0], isAnomalous: false }) store.saveCentroid({ fingerprint: 'fp-stale', dim: 2, vectorCount: 2, mean: [0, 0], isAnomalous: false }) const report = store.vectorStats('fp-active') assert.deepEqual(report.blocks, { total: 4, with_vector: 3, no_vector: 1 }) assert.deepEqual(report.spaces, { active: { fingerprint: 'fp-active', dim: 3 }, by_fingerprint: [ { fingerprint: 'fp-active', dim: 3, blocks: 1, status: 'active' }, { fingerprint: 'fp-stale', dim: 2, blocks: 2, status: 'stale' }, ], no_fingerprint: 0, }) return 'active=1 block/3D, stale=2 blocks/2D, no_vector=1; dimensions came from centroids' } finally { store.close() } }) // ── F8: explicit width errors, provider width errors degrade ───────────── await checkpoint('F8', 'dimension validation separates caller errors from provider degradation', async () => { const explicitApp = makeApp() const explicit = applyMemory(explicitApp.ctx, { path: join(workspace, 'f8-explicit.db'), embedder: { info: () => ({ fingerprint: 'fp-f8', dimension: 3 }), embedText: async () => [1, 0, 0] }, }) try { await assert.rejects( () => explicit.commit({ text: 'F8 caller error', vector: [1, 2], fingerprint: 'fp-f8' }), /does not match the active space/, ) assert.equal(explicit.store.countBlocks(), 0, 'caller width error must not write a block') await explicit.commit({ text: 'F8 matching caller vector', vector: [1, 2, 3], fingerprint: 'fp-f8' }) assert.equal(explicit.store.countBlocks(), 1, 'matching explicit vector is accepted') } finally { await explicitApp.ctx.fiber.dispose() } const providerApp = makeApp() const provider = applyMemory(providerApp.ctx, { path: join(workspace, 'f8-provider.db'), embedder: { info: () => ({ fingerprint: 'fp-f8-provider', dimension: 3 }), embedText: async () => [1, 2] }, }) try { const result = await provider.commit({ text: 'F8 provider error' }) assert.equal(result.embedded, false, 'provider width error must degrade without throwing') assert.equal(provider.store.countBlocks(), 1, 'degraded provider write still persists the block') assert.equal(provider.store.allBlocks(1)[0].vector, '', 'the bad provider vector is not stored') } finally { await providerApp.ctx.fiber.dispose() } return 'caller wrong width: rejected/no row; matching width: accepted; provider wrong width: degraded/row retained/vector empty' }) // ── cleanup ─────────────────────────────────────────────────────────────── rmSync(workspace, { recursive: true, force: true }) const passed = results.filter((entry) => entry.ok).length console.log(`\n${passed}/${results.length} checkpoints passed\n`) console.log(`plugin: name=${pluginName} inject=[${pluginInject.join(', ')}]`) console.log(`workspace: ${workspace} (removed)\n`) if (results.length === 0) { console.error('no checkpoints were registered — did the helper get renamed or stubbed?\n') process.exitCode = 1 } if (failed > 0) { console.error(`${failed} checkpoint(s) FAILED\n`) process.exitCode = 1 }