diff --git a/modules/story-summary/generate/prompt.js b/modules/story-summary/generate/prompt.js index 46b66fb..bfb95c3 100644 --- a/modules/story-summary/generate/prompt.js +++ b/modules/story-summary/generate/prompt.js @@ -91,9 +91,9 @@ function cleanSummary(summary) { function buildSystemPreamble() { return [ - "以上内容为因上下文窗口限制保留的可见历史", - "以下【剧情记忆】是对可见与不可见历史的总结:", - "• 【世界约束】记录着已确立的事实", + "以上是还留在眼前的对话", + "以下是脑海里的记忆:", + "• [定了的事] 这些是不会变的", "• 其余部分是过往经历的回忆碎片", "", "请内化这些记忆:", @@ -103,7 +103,7 @@ function buildSystemPreamble() { function buildPostscript() { return [ "", - "——", + "这些记忆是真实的,请自然地记住它们。", ].join("\n"); } @@ -594,49 +594,36 @@ async function buildVectorPrompt(store, recallResult, causalById, queryEntities } // ═══════════════════════════════════════════════════════════════════ - // 按注入顺序拼接 sections - // ═══════════════════════════════════════════════════════════════════ + // ═══════════════════════════════════════════════════════════════════════ +// 按注入顺序拼接 sections +// ═══════════════════════════════════════════════════════════════════════ +const sections = []; +// 1. 世界约束 → 定了的事 +if (assembled.world.lines.length) { + sections.push(`[定了的事] 已确立的事实\n${assembled.world.lines.join("\n")}`); +} +// 2. 核心经历 → 印象深的事 +if (assembled.events.direct.length) { + sections.push(`[印象深的事] 记得很清楚\n\n${assembled.events.direct.join("\n\n")}`); +} +// 3. 过往背景 → 好像有关的事 +if (assembled.events.similar.length) { + sections.push(`[好像有关的事] 听说过或有点模糊\n\n${assembled.events.similar.join("\n\n")}`); +} +// 4. 远期片段 → 更早以前 +if (assembled.orphans.lines.length) { + sections.push(`[更早以前] 记忆里残留的老画面\n${assembled.orphans.lines.join("\n")}`); +} +// 5. 待整理 → 刚发生的 +if (assembled.recentOrphans.lines.length) { + sections.push(`[刚发生的] 还没来得及想明白\n${assembled.recentOrphans.lines.join("\n")}`); +} +// 6. 人物弧光 → 这些人 +if (assembled.arcs.lines.length) { + sections.push(`[这些人] 他们现在怎样了\n${assembled.arcs.lines.join("\n")}`); +} - const sections = []; - - // 1. 世界约束 - if (assembled.world.lines.length) { - sections.push(`[世界约束] 已确立的事实\n${assembled.world.lines.join("\n")}`); - } - - // 2. 核心经历 - if (assembled.events.direct.length) { - sections.push(`[核心经历] 深刻的记忆\n\n${assembled.events.direct.join("\n\n")}`); - } - - // 3. 过往背景 - if (assembled.events.similar.length) { - sections.push(`[过往背景] 听别人说起或比较模糊的往事\n\n${assembled.events.similar.join("\n\n")}`); - } - - // 4. 远期片段 - if (assembled.orphans.lines.length) { - sections.push(`[远期片段] 记忆里残留的一些老画面\n${assembled.orphans.lines.join("\n")}`); - } - - // 5. 待整理 - if (assembled.recentOrphans.lines.length) { - sections.push(`[待整理] 最近发生但尚未梳理的原始记忆\n${assembled.recentOrphans.lines.join("\n")}`); - } - - // 6. 人物弧光(最后注入,但预算已在优先级 2 预留) - if (assembled.arcs.lines.length) { - sections.push(`[人物弧光]\n${assembled.arcs.lines.join("\n")}`); - } - - // ═══════════════════════════════════════════════════════════════════ - // 统计 & 返回 - // ═══════════════════════════════════════════════════════════════════ - - // 总预算 = 主装配 + 待整理 - injectionStats.budget.used = total.used + (recentOrphanStats.tokens || 0); - - if (!sections.length) { +if (!sections.length) { return { promptText: "", injectionLogText: "", injectionStats }; } @@ -848,4 +835,4 @@ export async function buildVectorPromptText(excludeLastAi = false, hooks = {}) { } return { text: finalText, logText: (recallResult.logText || "") + (injectionLogText || "") }; -} +} \ No newline at end of file diff --git a/modules/story-summary/vector/recall.js b/modules/story-summary/vector/recall.js index bf35f69..9a250db 100644 --- a/modules/story-summary/vector/recall.js +++ b/modules/story-summary/vector/recall.js @@ -296,19 +296,34 @@ function buildEntityLexicon(store, allEvents) { .slice(0, 5000); } -function extractEntities(text, lexicon) { - const t = normalize(text); - if (!t || !lexicon?.length) return []; +/** + * 从分段消息中提取实体,继承消息权重 + * @param {string[]} segments + * @param {number[]} weights + * @param {string[]} lexicon + * @returns {Map} + */ +function extractEntitiesWithWeights(segments, weights, lexicon) { + const entityWeights = new Map(); - const sorted = [...lexicon].sort((a, b) => b.length - a.length); - const hits = []; - for (const e of sorted) { - if (t.includes(e)) hits.push(e); - if (hits.length >= 20) break; + if (!segments?.length || !lexicon?.length) return entityWeights; + + for (let i = 0; i < segments.length; i++) { + const text = normalize(segments[i]); + const weight = weights?.[i] || 0; + + for (const entity of lexicon) { + if (text.includes(entity)) { + const existing = entityWeights.get(entity) || 0; + if (weight > existing) { + entityWeights.set(entity, weight); + } + } + } } - return hits; -} + return entityWeights; +} // ═══════════════════════════════════════════════════════════════════════════ // MMR // ═══════════════════════════════════════════════════════════════════════════ @@ -457,7 +472,7 @@ async function searchChunks(queryVector, vectorConfig, l0FloorBonus = new Map(), // L2 Events 检索(RRF 混合 + MMR 后置) // ═══════════════════════════════════════════════════════════════════════════ -async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus = new Map()) { +async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus = new Map()) { const { chatId } = getContext(); if (!chatId || !queryVector?.length) return []; @@ -475,11 +490,14 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo // 文本路检索 const textRanked = searchEventsByText(queryTextForSearch, CONFIG.TEXT_SEARCH_LIMIT); + const textGapInfo = textRanked._gapInfo || null; // ═══════════════════════════════════════════════════════════════════════ // 向量路检索(只保留 L0 加权) // ═══════════════════════════════════════════════════════════════════════ + const ENTITY_BONUS_FACTOR = 0.10; + const scored = (allEvents || []).map((event, idx) => { const v = vectorMap.get(event.id); const sim = v ? cosineSimilarity(queryVector, v) : 0; @@ -497,6 +515,17 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo } } + const participants = (event.participants || []).map(p => normalize(p)); + let maxEntityWeight = 0; + for (const p of participants) { + const w = queryEntityWeights.get(p) || 0; + if (w > maxEntityWeight) { + maxEntityWeight = w; + } + } + const entityBonus = ENTITY_BONUS_FACTOR * maxEntityWeight; + bonus += entityBonus; + return { _id: event.id, _idx: idx, @@ -504,9 +533,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo similarity: sim, finalScore: sim + bonus, vector: v, + _entityBonus: entityBonus, }; }); + const entityBonusById = new Map(scored.map(s => [s._id, s._entityBonus])); + const preFilterDistribution = { total: scored.length, '0.85+': scored.filter(s => s.finalScore >= 0.85).length, @@ -518,7 +550,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo threshold: CONFIG.MIN_SIMILARITY_EVENT, }; - // 向量路:纯相似度排序(不在这里做 MMR) const candidates = scored .filter(s => s.finalScore >= CONFIG.MIN_SIMILARITY_EVENT) .sort((a, b) => b.finalScore - a.finalScore) @@ -530,15 +561,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo vector: s.vector, })); - // RRF 融合 const eventById = new Map(allEvents.map(e => [e.id, e])); const fused = fuseEventsByRRF(vectorRanked, textRanked, eventById); - // 向量非空时过滤纯 TEXT const hasVector = vectorRanked.length > 0; const filtered = hasVector ? fused.filter(x => x.type !== 'TEXT') : fused; - // MMR 放在融合后:对最终候选集去重 const mmrInput = filtered.slice(0, CONFIG.CANDIDATE_EVENTS).map(x => ({ ...x, _id: x.id, @@ -551,7 +579,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo c => c.vector || null, c => c.rrf ); - // 构造结果 const results = mmrOutput.map(x => ({ event: x.event, @@ -559,6 +586,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo _recallType: x.type === 'HYBRID' ? 'DIRECT' : 'SIMILAR', _recallReason: x.type, _rrfDetail: { vRank: x.vRank, tRank: x.tRank, rrf: x.rrf }, + _entityBonus: entityBonusById.get(x.event?.id) || 0, })); // 统计信息附加到第一条结果 @@ -571,6 +599,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo vectorOnlyCount: fused.filter(x => x.type === 'VECTOR').length, textOnlyFiltered: fused.filter(x => x.type === 'TEXT').length, }; + results[0]._textGapInfo = textGapInfo; } return results; @@ -587,10 +616,11 @@ function formatRecallLog({ chunkResults, eventResults, allEvents, - queryEntities, + queryEntityWeights = new Map(), causalEvents = [], chunkPreFilterStats = null, l0Results = [], + textGapInfo = null, }) { const lines = [ '\u2554' + '\u2550'.repeat(62) + '\u2557', @@ -621,7 +651,18 @@ function formatRecallLog({ lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510'); lines.push('\u2502 【提取实体】 \u2502'); lines.push('\u2514' + '\u2500'.repeat(61) + '\u2518'); - lines.push(` ${queryEntities?.length ? queryEntities.join('、') : '(无)'}`); + + if (queryEntityWeights?.size) { + const sorted = Array.from(queryEntityWeights.entries()) + .sort((a, b) => b[1] - a[1]) + .slice(0, 8); + const formatted = sorted + .map(([e, w]) => `${e}(${(w * 100).toFixed(0)}%)`) + .join(' | '); + lines.push(` ${formatted}`); + } else { + lines.push(' (无)'); + } lines.push(''); lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510'); @@ -642,7 +683,7 @@ function formatRecallLog({ lines.push(' L1 原文片段:'); if (chunkPreFilterStats) { const dist = chunkPreFilterStats.distribution || {}; - lines.push(` \u5168\u91cf: ${chunkPreFilterStats.total} \u6761 | \u901a\u8fc7\u9608\u503c(\u8fdc\u671f\u2265${chunkPreFilterStats.thresholdRemote}, \u5f85\u6574\u7406\u2265${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} \u6761 | \u6700\u7ec8: ${chunkResults.length} \u6761`); + lines.push(` 全量: ${chunkPreFilterStats.total} 条 | 通过阈值(远期≥${chunkPreFilterStats.thresholdRemote}, 待整理≥${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} 条 | 最终: ${chunkResults.length} 条`); lines.push(` 匹配度: 0.8+: ${dist['0.8+'] || 0} | 0.7-0.8: ${dist['0.7-0.8'] || 0} | 0.6-0.7: ${dist['0.6-0.7'] || 0}`); } else { lines.push(` 选入: ${chunkResults.length} 条`); @@ -656,6 +697,18 @@ function formatRecallLog({ lines.push(` 总事件: ${allEvents.length} 条 | 最终: ${eventResults.length} 条`); lines.push(` 向量路: ${rrfStats.vectorCount || 0} 条 | 文本路: ${rrfStats.textCount || 0} 条`); lines.push(` HYBRID: ${rrfStats.hybridCount || 0} 条 | 纯 VECTOR: ${rrfStats.vectorOnlyCount || 0} 条 | 纯 TEXT (已过滤): ${rrfStats.textOnlyFiltered || 0} 条`); + const entityBoostedEvents = eventResults.filter(e => e._entityBonus > 0).length; + lines.push(` 实体加分事件: ${entityBoostedEvents} 条`); + + if (textGapInfo) { + lines.push(''); + lines.push(' 文本检索 (BM25 动态 top-K):'); + lines.push(` 命中: ${textGapInfo.total} 条 | 返回: ${textGapInfo.returned} 条 (覆盖 ${textGapInfo.coverage} 总分)`); + if (textGapInfo.scoreRange) { + const s = textGapInfo.scoreRange; + lines.push(` 分数: Top=${s.top} | 截断=${s.cutoff} | P50=${s.p50} | Last=${s.last}`); + } + } // Causal if (causalEvents.length) { @@ -702,7 +755,8 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = } const lexicon = buildEntityLexicon(store, allEvents); - const queryEntities = extractEntities(segments.join('\n'), lexicon); + const queryEntityWeights = extractEntitiesWithWeights(segments, weights, lexicon); + const queryEntities = Array.from(queryEntityWeights.keys()); // 构建文本查询串:最后一条消息 + 实体 + 关键词 const lastSeg = segments[segments.length - 1] || ''; @@ -727,10 +781,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = const [chunkResults, eventResults] = await Promise.all([ searchChunks(queryVector, vectorConfig, l0FloorBonus, lastSummarizedFloor), - searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus), + searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus), ]); const chunkPreFilterStats = chunkResults._preFilterStats || null; + const textGapInfo = eventResults[0]?._textGapInfo || null; const mergedChunks = mergeAndSparsify(l0VirtualChunks, chunkResults, CONFIG.FLOOR_MAX_CHUNKS); @@ -764,10 +819,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options = chunkResults: mergedChunks, eventResults, allEvents, - queryEntities, + queryEntityWeights, causalEvents: causalEventsTruncated, chunkPreFilterStats, l0Results, + textGapInfo, }); console.group('%c[Recall]', 'color: #7c3aed; font-weight: bold'); diff --git a/modules/story-summary/vector/text-search.js b/modules/story-summary/vector/text-search.js index 3419216..d173c56 100644 --- a/modules/story-summary/vector/text-search.js +++ b/modules/story-summary/vector/text-search.js @@ -1,37 +1,70 @@ -// ═══════════════════════════════════════════════════════════════════════════ -// Text Search - L2 事件文本检索(MiniSearch) -// 与向量检索互补,通过 RRF 融合 -// ═══════════════════════════════════════════════════════════════════════════ +// text-search.js - 最终版 import MiniSearch from '../../../libs/minisearch.mjs'; +const STOP_WORDS = new Set([ + '的', '了', '是', '在', '和', '与', '或', '但', '而', '却', + '这', '那', '他', '她', '它', '我', '你', '们', '着', '过', + '把', '被', '给', '让', '向', '就', '都', '也', '还', '又', + '很', '太', '更', '最', '只', '才', '已', '正', '会', '能', + '要', '可', '得', '地', '之', '所', '以', '为', '于', '有', + '不', '去', '来', '上', '下', '里', '说', '看', '吧', '呢', + '啊', '吗', '呀', '哦', '嗯', '么', + 'の', 'に', 'は', 'を', 'が', 'と', 'で', 'へ', 'や', 'か', + 'も', 'な', 'よ', 'ね', 'わ', 'です', 'ます', 'した', 'ない', + 'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', + 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', + 'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from', + 'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its', + 'i', 'you', 'he', 'she', 'we', 'they', 'my', 'your', 'his', +]); + +function tokenize(text) { + const s = String(text || '').toLowerCase().trim(); + if (!s) return []; + + const tokens = new Set(); + + // CJK Bigram + Trigram + const cjk = s.match(/[\u4e00-\u9fff\u3400-\u4dbf]+/g) || []; + for (const seg of cjk) { + const chars = [...seg].filter(c => !STOP_WORDS.has(c)); + for (let i = 0; i < chars.length - 1; i++) { + tokens.add(chars[i] + chars[i + 1]); + } + for (let i = 0; i < chars.length - 2; i++) { + tokens.add(chars[i] + chars[i + 1] + chars[i + 2]); + } + } + + // 日语假名 + const kana = s.match(/[\u3040-\u309f\u30a0-\u30ff]{2,}/g) || []; + for (const k of kana) { + if (!STOP_WORDS.has(k)) tokens.add(k); + } + + // 英文 + const en = s.match(/[a-z]{2,}/g) || []; + for (const w of en) { + if (!STOP_WORDS.has(w)) tokens.add(w); + } + + return [...tokens]; +} + let idx = null; let lastRevision = null; -/** - * 中文逐字 + 英数字串分词 - */ -function tokenize(text) { - return String(text || '').match(/[\u4e00-\u9fff]|[a-zA-Z0-9]+/g) || []; -} - -/** - * 去掉 summary 末尾的楼层标记 - */ function stripFloorTag(s) { return String(s || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').trim(); } -/** - * 构建/更新事件文本索引 - */ export function ensureEventTextIndex(events, revision) { if (!events?.length) { idx = null; lastRevision = null; return; } - if (idx && revision === lastRevision) return; try { @@ -39,6 +72,7 @@ export function ensureEventTextIndex(events, revision) { fields: ['title', 'summary', 'participants'], storeFields: ['id'], tokenize, + searchOptions: { tokenize }, }); idx.addAll(events.map(e => ({ @@ -52,33 +86,87 @@ export function ensureEventTextIndex(events, revision) { } catch (e) { console.error('[text-search] Index build failed:', e); idx = null; - lastRevision = null; } } /** - * 文本检索事件 + * BM25 检索,返回 top-K 候选给 RRF + * + * 设计原则: + * - 不做分数过滤(BM25 分数跨查询不可比) + * - 不做匹配数过滤(bigram 让一个词产生多个 token) + * - 只做 top-K(BM25 排序本身有区分度) + * - 质量过滤交给 RRF 后的 hasVector 过滤 */ +/** + * 动态 top-K:累积分数占比法 + * + * 原理:BM25 分数服从幂律分布,少数高分条目贡献大部分总分 + * 取累积分数达到阈值的最小 K + * + * 参考:帕累托法则(80/20 法则)在信息检索中的应用 + */ +function dynamicTopK(scores, coverage = 0.90, minK = 15, maxK = 80) { + if (!scores.length) return 0; + + const total = scores.reduce((a, b) => a + b, 0); + if (total <= 0) return Math.min(minK, scores.length); + + let cumulative = 0; + for (let i = 0; i < scores.length; i++) { + cumulative += scores[i]; + if (cumulative / total >= coverage) { + return Math.max(minK, Math.min(maxK, i + 1)); + } + } + + return Math.min(maxK, scores.length); +} + export function searchEventsByText(queryText, limit = 80) { if (!idx || !queryText?.trim()) return []; try { - const res = idx.search(queryText, { - limit, - boost: { title: 2, participants: 1.5, summary: 1 }, - fuzzy: 0.2, - prefix: true, + const results = idx.search(queryText, { + boost: { title: 4, participants: 2, summary: 1 }, + fuzzy: false, + prefix: false, }); - return res.map((r, i) => ({ id: r.id, textRank: i + 1 })); + + if (!results.length) return []; + + const scores = results.map(r => r.score); + const k = dynamicTopK(scores, 0.90, 15, limit); + + const output = results.slice(0, k).map((r, i) => ({ + id: r.id, + textRank: i + 1, + score: r.score, + })); + + const total = scores.reduce((a, b) => a + b, 0); + const kCumulative = scores.slice(0, k).reduce((a, b) => a + b, 0); + + output._gapInfo = { + total: results.length, + returned: k, + coverage: ((kCumulative / total) * 100).toFixed(1) + '%', + scoreRange: { + top: scores[0]?.toFixed(1), + cutoff: scores[k - 1]?.toFixed(1), + p50: scores[Math.floor(scores.length / 2)]?.toFixed(1), + last: scores[scores.length - 1]?.toFixed(1), + }, + }; + + return output; + } catch (e) { console.error('[text-search] Search failed:', e); return []; } } -/** - * 清理索引 - */ export function clearEventTextIndex() { idx = null; lastRevision = null;