diff --git a/lib/index.js b/lib/index.js index a265456..d7c6645 100644 --- a/lib/index.js +++ b/lib/index.js @@ -296,7 +296,8 @@ export class StableMem extends Service { * disagree — the failure mode D4 exists to prevent. * * @type {{mode: string, skipped_fp: number, skipped_dim: number, - * skipped_score: number, skipped_no_evidence: number, truncated: number, + * skipped_score: number, skipped_no_evidence: number, + * skipped_unscoreable: number, truncated: number, * embed_error: string|null, at: number} | null} */ lastRecall = null @@ -567,6 +568,11 @@ export class StableMem extends Service { skipped_dim: hits.skippedDim ?? 0, skipped_score: hits.skippedScore ?? 0, skipped_no_evidence: hits.skippedNoEvidence ?? 0, + // D2's second hole: blocks dropped BEFORE they could enter the candidate + // pool (bad JSON, wrong width, non-finite component, zero norm, or — since + // C4 — a vector that centring collapses onto the space mean). Those are not + // "gated": no gate ever saw them, which is why they were invisible. + skipped_unscoreable: hits.skippedUnscoreable ?? 0, truncated: hits.truncated ?? 0, embed_error: hits.degradedError ?? null, at: Date.now(), @@ -681,7 +687,9 @@ export class StableMem extends Service { raw?.skippedDim ?? 0 } skippedScore=${raw?.skippedScore ?? 0} skippedNoEvidence=${ raw?.skippedNoEvidence ?? 0 - } truncated=${raw?.truncated ?? 0} err=${summarizeError(raw?.degradedError)}`, + } skippedUnscoreable=${raw?.skippedUnscoreable ?? 0} truncated=${ + raw?.truncated ?? 0 + } err=${summarizeError(raw?.degradedError)}`, `inject=${injectChars}ch`, ].join(' | ') diff --git a/lib/recall.js b/lib/recall.js index 1a7c763..4fcdc47 100644 --- a/lib/recall.js +++ b/lib/recall.js @@ -254,7 +254,7 @@ function allIndexesOf(haystack, needle) { * @returns {Array<{id: string, text: string, source: string, score: number, * exact: boolean, why: string}>} */ -export function recallBlocks(store, query, { topK = 5, poolSize = 50, fingerprint, dimension, vectorScores, minScore } = {}) { +export function recallBlocks(store, query, { topK = 5, poolSize = 50, fingerprint, dimension, vectorScores, minScore, skippedUnscoreable = 0 } = {}) { const text = String(query ?? '').trim() if (text === '') return [] @@ -554,7 +554,10 @@ export function recallBlocks(store, query, { topK = 5, poolSize = 50, fingerprin scored.sort((a, b) => (b.score === a.score ? a.id.localeCompare(b.id) : b.score - a.score)) // ported from go: internal/memory/fusion.go:557-559 — final caller topK, // after candidate union, scoring, filtering and the asymmetric gate. - const drops = { gatedByScore, gatedNoEvidence, truncated: Math.max(0, scored.length - topK) } + const drops = { + gatedByScore, gatedNoEvidence, skippedUnscoreable, + truncated: Math.max(0, scored.length - topK), + } return withSkipped(scored.slice(0, topK), skipped, skippedDim, drops) } @@ -599,6 +602,9 @@ function withSkipped(hits, skipped, skippedDim = 0, drops = {}) { ['gatedByScore', 'skippedScore'], ['gatedNoEvidence', 'skippedNoEvidence'], ['truncated', 'truncated'], + // D2's second hole: blocks dropped BEFORE the candidate pool existed. They + // are not "gated" (no gate ever saw them) — they could not be scored at all. + ['skippedUnscoreable', 'skippedUnscoreable'], ]) { Object.defineProperty(hits, label, { value: Number.isInteger(drops[key]) ? drops[key] : 0, @@ -797,14 +803,34 @@ export async function recallBlocksHybrid(store, embedder, query, options = {}) { // step, `allBlocks(-1)`), so the fused path was always O(total blocks); what is // added here is one narrow indexed read plus the cosine arithmetic, and the // decode is bounded to the active space and width instead of the whole table. - const vectorScores = scoreVectorCandidates(store, queryVector, scoped) + // ── D2: blocks that could not be scored at all ────────────────────────── + // + // `scoreVectorCandidates` skips a row when it cannot produce a number: bad + // JSON, wrong width, a non-finite component, a zero-norm vector, or — since C4 + // — a vector that centring collapses to dust. Such a block never enters + // `vectorScores`, so `recallBlocks` never puts it in the candidate pool, so + // none of the downstream counters can see it. + // + // Measured: three identical blocks with centring active produced + // `hits=0 skipped=0 skippedDim=0 skippedScore=0 skippedNoEvidence=0 truncated=0` + // — the whole space vanished while every counter read zero. That is exactly the + // silent branch D2 forbids, one stage EARLIER than where D2 first looked. + // + // The counter is threaded by reference so it can be reported on both the fused + // path and the early `no_vector_space` return. + const unscoreable = { count: 0 } + + const vectorScores = scoreVectorCandidates(store, queryVector, scoped, unscoreable) const vectorRoute = vectorScores.size > 0 if (!vectorRoute) { // An empty map would still open the fused candidate assembly (minScore is // finite) and then admit symbol matches with a vector weight of 0 — the // `0.7*0 + 0.3*coverage` regression V2 forbids. Reporting the honest reason // and taking the untouched symbol path is both correct and cheaper. - return withDiagnosis(recallBlocks(store, query, scoped), 'no_vector_space') + return withDiagnosis( + recallBlocks(store, query, { ...scoped, skippedUnscoreable: unscoreable.count }), + 'no_vector_space', + ) } // `minScore` is only defaulted when the caller did not decide: an explicit @@ -813,7 +839,7 @@ export async function recallBlocksHybrid(store, embedder, query, options = {}) { const minScore = Number.isFinite(scoped.minScore) ? scoped.minScore : DEFAULT_MIN_SCORE return withDiagnosis( - recallBlocks(store, query, { ...scoped, vectorScores, minScore }), + recallBlocks(store, query, { ...scoped, vectorScores, minScore, skippedUnscoreable: unscoreable.count }), // 'hybrid' from here on is a statement of fact, not an aspiration: the vector // route contributed at least one scoreable candidate to the same call that // produced these hits. Group A shipped `vector_ready_unused` precisely so this @@ -848,7 +874,7 @@ export async function recallBlocksHybrid(store, embedder, query, options = {}) { * @param {{fingerprint?: string, dimension?: number}} options * @returns {Map} */ -function scoreVectorCandidates(store, queryVector, { fingerprint, dimension } = {}) { +function scoreVectorCandidates(store, queryVector, { fingerprint, dimension } = {}, unscoreable = { count: 0 }) { const scores = new Map() // Without a space identity there is nothing to compare against: the vectors in // the table may belong to several incompatible spaces (§0.1), and a cosine @@ -893,16 +919,28 @@ function scoreVectorCandidates(store, queryVector, { fingerprint, dimension } = try { vector = JSON.parse(String(row.vector ?? '')) } catch { + unscoreable.count += 1 + continue + } + if (!Array.isArray(vector) || vector.length !== queryVector.length) { + unscoreable.count += 1 continue } - if (!Array.isArray(vector) || vector.length !== queryVector.length) continue // ★ 两边都中心化。只处理一边 = 一边带公共分量、另一边不带,算出来的 // 余弦是两种量纲的混合(HA centroid_store.go:439-442 的原话)。 // 实测:只减 query 时,C4b 那个 5 块 fixture 上同值对的余弦从 1.0 掉到 // 0.1897(余弦对称,只减块侧同值)—— 那就不是在量夹角了。 const scoringVector = centering ? centerVector(vector, mean) : vector - if (scoringVector === null) continue + if (scoringVector === null) { + // ★ C4's dust guard fired: the block sits on the space mean, so it has no + // direction to compare against. Counted explicitly, because otherwise this + // block disappears BEFORE the candidate pool is assembled and no downstream + // counter can see it — measured as "the whole space vanished while every + // counter read zero". + unscoreable.count += 1 + continue + } let dot = 0 let norm = 0 @@ -918,7 +956,10 @@ function scoreVectorCandidates(store, queryVector, { fingerprint, dimension } = dot += value * scoringQuery[index] norm += value * value } - if (!Number.isFinite(dot) || !(norm > 0)) continue + if (!Number.isFinite(dot) || !(norm > 0)) { + unscoreable.count += 1 + continue + } scores.set(row.id, dot / (queryNorm * Math.sqrt(norm))) } return scores diff --git a/test/memory-log.test.js b/test/memory-log.test.js index 436d485..a631c08 100644 --- a/test/memory-log.test.js +++ b/test/memory-log.test.js @@ -172,7 +172,7 @@ test('L2: debugLog=true emits one line covering recall, budget, degradation and assert.match(record.text, /skipped=0 skippedDim=0/, 'skip counters') // D2: the three downstream counters must be PRINTED, not merely computed. // A counter nobody can read is indistinguishable from no counter at all. - assert.match(record.text, /skippedScore=0 skippedNoEvidence=0 truncated=0/, 'downstream drop counters') + assert.match(record.text, /skippedScore=0 skippedNoEvidence=0 skippedUnscoreable=0 truncated=0/, 'downstream drop counters') assert.match(record.text, /err=-/, 'no error digest when there is no error') // injection: the exact string length that reached the model. @@ -250,7 +250,7 @@ test('L2b: non-zero drop counters appear in the log and in last_recall', async ( assert.equal(emitted.length, 1, `exactly one line per recall turn; got ${JSON.stringify(emitted)}`) assert.match( emitted[0].text, - /skippedScore=1 skippedNoEvidence=1 truncated=0/, + /skippedScore=1 skippedNoEvidence=1 skippedUnscoreable=0 truncated=0/, `D2: the counters must survive segmenting and reach the log; got ${emitted[0].text}`, ) assert.equal(mem.lastRecall.skipped_score, 1, 'and the durable surface must agree') diff --git a/test/recall-accounting.test.js b/test/recall-accounting.test.js index f7f3a9a..92077bc 100644 --- a/test/recall-accounting.test.js +++ b/test/recall-accounting.test.js @@ -29,7 +29,7 @@ import assert from 'node:assert/strict' import test from 'node:test' import { BlockStore } from '../lib/store.js' -import { recallBlocks } from '../lib/recall.js' +import { recallBlocks, recallBlocksHybrid } from '../lib/recall.js' const DIM = 8 const FP = 'd2:accounting' @@ -118,6 +118,7 @@ test('D2: hits plus every counter reconciles with the candidate pool', () => { const accounted = hits.length + hits.skippedScore + hits.skippedNoEvidence + + hits.skippedUnscoreable + hits.truncated + hits.skipped + hits.skippedDim @@ -131,3 +132,60 @@ test('D2: hits plus every counter reconciles with the candidate pool', () => { store.close() } }) + +// ── D2 的第二个洞:在候选池存在之前就消失的块 ────────────────────────────── +// +// 07f8438 给三个「门」加了计数,但候选池是三处之外还有一段:`scoreVectorCandidates` +// 里算不出分数的行直接 continue —— 坏 JSON、宽度不符、非有限分量、零模,以及 +// C4 之后新增的一种:**中心化把向量抹成尘埃**。这些块压根不进 vectorScores, +// 于是 recallBlocks 装配候选池时(`if (!symbolMatch && !vectorSource) continue`) +// 也看不到它们。 +// +// 实测(3 条完全相同的块 + 中心化生效): +// hits=0 skipped=0 skippedDim=0 skippedScore=0 skippedNoEvidence=0 truncated=0 +// —— 整个空间凭空消失,六个计数器全读 0。这正是 D2 自己禁止的静默分支, +// 只是位置比当初看的更深一层。 +// +// 这条判据同时钉住两件事:计数非零,以及「命中 + 全部计数」仍能对上候选池。 +test('D2: a vector that centring collapses is counted, not silently lost', async () => { + const DIM = 4 + const FP = 'd2:unscoreable' + const unit = (v) => { const n = Math.hypot(...v); return n === 0 ? v : v.map((x) => x / n) } + const blockVec = unit([1, 0.2, 0.1, 0]) + const queryVec = unit([1, 0.9, 0.1, 0]) + + const store = new BlockStore({ path: ':memory:' }).open() + try { + for (let i = 0; i < 3; i += 1) { + store.putBlock({ text: `重复块 ${i}`, vector: blockVec, fingerprint: FP }) + } + const built = store.rebuildCentroid(FP) + assert.ok(built.meanLength > 0.999, `precondition: identical blocks give a unit mean; got ${built.meanLength}`) + assert.equal(built.isAnomalous, true, 'precondition: centring must actually engage') + + const embedder = { + async embedText() { return queryVec }, + info: () => ({ fingerprint: FP, dimension: DIM }), + } + const hits = await recallBlocksHybrid(store, embedder, 'zzz-no-symbol-match', { topK: 10, minScore: 0 }) + + assert.equal(hits.length, 0, 'precondition: every block is centred onto the mean and dropped') + assert.equal( + hits.skippedUnscoreable, + 3, + 'D2: these blocks were dropped before the candidate pool existed, so no other ' + + 'counter can see them. Reporting 0 here is what made a whole space vanish ' + + 'while the log read "skipped=0".', + ) + + const accounted = hits.length + hits.skipped + hits.skippedDim + hits.skippedScore + + hits.skippedNoEvidence + hits.skippedUnscoreable + hits.truncated + assert.equal( + accounted, + 3, + `D2: hits + every counter must equal the space's blocks; ${accounted} of 3.`, + ) + } finally { + store.close() + } +})