HologramAgent-DSH-Memory-Port/test/recall-accounting.test.js

/**
 * D2 · 每一次丢弃都必须可计数。
 *
 * ## 病
 *
 * 一条候选可以从三个地方消失，而在本次改动之前只有前两个（空间过滤、宽度
 * 不符）会被报告：
 *
 *   · MinScore 门        —— 它自己的向量分不够
 *   · 结构性门           —— 既没有可用的向量分，也没有符号命中
 *   · topK 截断          —— 过了所有门，但排在调用方要的条数之外
 *
 * 后三个都是裸 `continue`，不进任何计数器。于是「6 条候选留下 1 条」与
 * 「库里只有 1 条」在对外读数上完全一样：`skipped=0 skippedDim=0`。
 * 实测（改动前）：6 条块、minScore=0.99 → hits=0，两个计数都是 0。
 *
 * 运维看 `skipped=0` 会得出「没有任何东西被排除」的结论，而真相可能是阈值
 * 把整批都挡了 —— 一个应该调阈值的信号被读成了「库是空的」。
 *
 * ## 判据形态
 *
 * 三条路径各造一个 fixture，断**各自**的计数。另加一条不变式：
 * 候选池 = 命中 + 三个下游计数 + 上游两个计数，等式必须成立 —— 否则将来再
 * 加一条静默丢弃路径，等式就会破。
 *
 * @module dsh-memory/test/recall-accounting
 */

import assert from 'node:assert/strict'
import test from 'node:test'
import { BlockStore } from '../lib/store.js'
import { recallBlocks, recallBlocksHybrid } from '../lib/recall.js'

const DIM = 8
const FP = 'd2:accounting'
const unit = (v) => { const n = Math.hypot(...v); return n === 0 ? v : v.map((x) => x / n) }
const axis = (i) => unit(Array.from({ length: DIM }, (_, k) => (k === i ? 1 : 0)))

/** 6 条块，各占一轴；query 落在轴 0，所以 cosine 已知且互不相同。 */
function seeded() {
  const store = new BlockStore({ path: ':memory:' }).open()
  const ids = []
  for (let i = 0; i < 6; i += 1) {
    ids.push(store.putBlock({ text: `块 ${i} 的正文`, vector: axis(i), fingerprint: FP }).id)
  }
  const query = unit([1, 0.5, 0.2, 0, 0, 0, 0, 0])
  const scores = new Map(ids.map((id, i) => [id, i === 0 ? 0.95 : 0.1 + i * 0.01]))
  return { store, ids, query, scores }
}

test('D2: candidates rejected by MinScore are counted, not silently dropped', () => {
  const { store, query, scores } = seeded()
  try {
    const hits = recallBlocks(store, query, { topK: 6, fingerprint: FP, dimension: DIM, vectorScores: scores, minScore: 0.9 })
    assert.equal(hits.length, 1, 'precondition: only the 0.95 candidate clears 0.9')
    assert.equal(
      hits.skippedScore,
      5,
      'D2: five candidates failed the score gate and must be REPORTED as such. '
        + 'Before this counter existed the result was indistinguishable from a '
        + 'one-block space, which sends the operator looking at the wrong subsystem.',
    )
  } finally {
    store.close()
  }
})

test('D2: candidates with neither vector score nor symbol hit are counted separately', () => {
  const { store, ids, query } = seeded()
  try {
    // A candidate reaches the STRUCTURAL gate when it IS in the pool but has no
    // usable vector score (absent, or <= 0) and no symbol match. A cosine of
    // exactly 0 is the realistic trigger: it is what a candidate orthogonal to
    // the query gets, and also what centring produces in a degenerate space.
    const scores = new Map([[ids[0], 0.95], [ids[1], 0.8], [ids[2], 0]])
    const hits = recallBlocks(store, 'zzz-no-symbol-match', {
      topK: 6, fingerprint: FP, dimension: DIM, vectorScores: scores, minScore: 0.5,
    })
    assert.equal(hits.length, 2, 'precondition: only the two positive scores survive')
    assert.equal(hits.skippedScore, 0, 'precondition: nothing failed the threshold')
    assert.equal(
      hits.skippedNoEvidence,
      1,
      'D2: a candidate that matched neither route must be counted under its own name — '
        + '"nothing to score" and "scored too low" have opposite remedies (re-embed vs. '
        + 'lower the threshold). Blocks that never entered the pool are a different, '
        + 'upstream fact and must not be folded in here.',
    )
  } finally {
    store.close()
  }
})

test('D2: candidates cut by the caller topK are counted as truncated', () => {
  const { store, query, scores } = seeded()
  try {
    const all = recallBlocks(store, query, { topK: 6, fingerprint: FP, dimension: DIM, vectorScores: scores, minScore: 0.05 })
    const top2 = recallBlocks(store, query, { topK: 2, fingerprint: FP, dimension: DIM, vectorScores: scores, minScore: 0.05 })
    assert.equal(all.length, 6, 'precondition: all six clear a 0.05 threshold')
    assert.equal(top2.length, 2, 'precondition: topK truncates')
    assert.equal(top2.skippedScore, 0, 'nothing failed the score gate here')
    assert.equal(
      top2.truncated,
      4,
      'D2: the four that fell outside topK must be reported. Truncation is not a '
        + 'defect, but leaving it uncounted breaks the reconciliation of '
        + 'candidates = hits + counters.',
    )
  } finally {
    store.close()
  }
})

test('D2: hits plus every counter reconciles with the candidate pool', () => {
  const { store, query, scores } = seeded()
  try {
    const hits = recallBlocks(store, query, { topK: 3, fingerprint: FP, dimension: DIM, vectorScores: scores, minScore: 0.15 })
    const accounted = hits.length
      + hits.skippedScore
      + hits.skippedNoEvidence
      + hits.skippedUnscoreable
      + hits.truncated
      + hits.skipped
      + hits.skippedDim
    assert.equal(
      accounted,
      6,
      `D2: every candidate must land in exactly one bucket; ${accounted} of 6 accounted for. `
        + 'A shortfall means a new silent drop path was added without a counter.',
    )
  } finally {
    store.close()
  }
})

// ── D2 的第二个洞：在候选池存在之前就消失的块 ──────────────────────────────
//
// 07f8438 给三个「门」加了计数，但候选池是三处之外还有一段：`scoreVectorCandidates`
// 里算不出分数的行直接 continue —— 坏 JSON、宽度不符、非有限分量、零模，以及
// C4 之后新增的一种：**中心化把向量抹成尘埃**。这些块压根不进 vectorScores，
// 于是 recallBlocks 装配候选池时（`if (!symbolMatch && !vectorSource) continue`）
// 也看不到它们。
//
// 实测（3 条完全相同的块 + 中心化生效）：
//   hits=0 skipped=0 skippedDim=0 skippedScore=0 skippedNoEvidence=0 truncated=0
//   —— 整个空间凭空消失，六个计数器全读 0。这正是 D2 自己禁止的静默分支，
//      只是位置比当初看的更深一层。
//
// 这条判据同时钉住两件事：计数非零，以及「命中 + 全部计数」仍能对上候选池。
test('D2: a vector that centring collapses is counted, not silently lost', async () => {
  const DIM = 4
  const FP = 'd2:unscoreable'
  const unit = (v) => { const n = Math.hypot(...v); return n === 0 ? v : v.map((x) => x / n) }
  const blockVec = unit([1, 0.2, 0.1, 0])
  const queryVec = unit([1, 0.9, 0.1, 0])

  const store = new BlockStore({ path: ':memory:' }).open()
  try {
    for (let i = 0; i < 3; i += 1) {
      store.putBlock({ text: `重复块 ${i}`, vector: blockVec, fingerprint: FP })
    }
    const built = store.rebuildCentroid(FP)
    assert.ok(built.meanLength > 0.999, `precondition: identical blocks give a unit mean; got ${built.meanLength}`)
    assert.equal(built.isAnomalous, true, 'precondition: centring must actually engage')

    const embedder = {
      async embedText() { return queryVec },
      info: () => ({ fingerprint: FP, dimension: DIM }),
    }
    const hits = await recallBlocksHybrid(store, embedder, 'zzz-no-symbol-match', { topK: 10, minScore: 0 })

    assert.equal(hits.length, 0, 'precondition: every block is centred onto the mean and dropped')
    assert.equal(
      hits.skippedUnscoreable,
      3,
      'D2: these blocks were dropped before the candidate pool existed, so no other '
        + 'counter can see them. Reporting 0 here is what made a whole space vanish '
        + 'while the log read "skipped=0".',
    )

    const accounted = hits.length + hits.skipped + hits.skippedDim + hits.skippedScore
      + hits.skippedNoEvidence + hits.skippedUnscoreable + hits.truncated
    assert.equal(
      accounted,
      3,
      `D2: hits + every counter must equal the space's blocks; ${accounted} of 3.`,
    )
  } finally {
    store.close()
  }
})