X/LittleWhiteBox
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update recall entity weighting and prompt sections

Browse Source
This commit is contained in:
bielie
2026-02-02 14:02:12 +08:00
parent d8849c5e8b
commit d3f772073f
3 changed files with 229 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
modules/story-summary/generate/prompt.js
View File

@@ -91,9 +91,9 @@ function cleanSummary(summary) {
function buildSystemPreamble() {
return [
"以上内容为因上下文窗口限制保留的可见历史",
"以下【剧情记忆】是对可见与不可见历史的总结：",
"• 【世界约束】记录着已确立的事实",
"以上是还留在眼前的对话",
"以下是脑海里的记忆：",
"• [定了的事] 这些是不会变的",
"• 其余部分是过往经历的回忆碎片",
"",
"请内化这些记忆：",
@@ -103,7 +103,7 @@ function buildSystemPreamble() {
function buildPostscript() {
return [
"",
"——",
"这些记忆是真实的，请自然地记住它们。",
].join("\n");
}
@@ -594,49 +594,36 @@ async function buildVectorPrompt(store, recallResult, causalById, queryEntities
}
// ═══════════════════════════════════════════════════════════════════
// 按注入顺序拼接 sections
// ═══════════════════════════════════════════════════════════════════
// ═══════════════════════════════════════════════════════════════════════
// 按注入顺序拼接 sections
// ═══════════════════════════════════════════════════════════════════════
const sections = [];
// 1. 世界约束 → 定了的事
if (assembled.world.lines.length) {
sections.push(`[定了的事] 已确立的事实\n${assembled.world.lines.join("\n")}`);
}
// 2. 核心经历 → 印象深的事
if (assembled.events.direct.length) {
sections.push(`[印象深的事] 记得很清楚\n\n${assembled.events.direct.join("\n\n")}`);
}
// 3. 过往背景 → 好像有关的事
if (assembled.events.similar.length) {
sections.push(`[好像有关的事] 听说过或有点模糊\n\n${assembled.events.similar.join("\n\n")}`);
}
// 4. 远期片段 → 更早以前
if (assembled.orphans.lines.length) {
sections.push(`[更早以前] 记忆里残留的老画面\n${assembled.orphans.lines.join("\n")}`);
}
// 5. 待整理 → 刚发生的
if (assembled.recentOrphans.lines.length) {
sections.push(`[刚发生的] 还没来得及想明白\n${assembled.recentOrphans.lines.join("\n")}`);
}
// 6. 人物弧光 → 这些人
if (assembled.arcs.lines.length) {
sections.push(`[这些人] 他们现在怎样了\n${assembled.arcs.lines.join("\n")}`);
}
const sections = [];
// 1. 世界约束
if (assembled.world.lines.length) {
sections.push(`[世界约束] 已确立的事实\n${assembled.world.lines.join("\n")}`);
}
// 2. 核心经历
if (assembled.events.direct.length) {
sections.push(`[核心经历] 深刻的记忆\n\n${assembled.events.direct.join("\n\n")}`);
}
// 3. 过往背景
if (assembled.events.similar.length) {
sections.push(`[过往背景] 听别人说起或比较模糊的往事\n\n${assembled.events.similar.join("\n\n")}`);
}
// 4. 远期片段
if (assembled.orphans.lines.length) {
sections.push(`[远期片段] 记忆里残留的一些老画面\n${assembled.orphans.lines.join("\n")}`);
}
// 5. 待整理
if (assembled.recentOrphans.lines.length) {
sections.push(`[待整理] 最近发生但尚未梳理的原始记忆\n${assembled.recentOrphans.lines.join("\n")}`);
}
// 6. 人物弧光（最后注入，但预算已在优先级 2 预留）
if (assembled.arcs.lines.length) {
sections.push(`[人物弧光]\n${assembled.arcs.lines.join("\n")}`);
}
// ═══════════════════════════════════════════════════════════════════
// 统计 & 返回
// ═══════════════════════════════════════════════════════════════════
// 总预算 = 主装配 + 待整理
injectionStats.budget.used = total.used + (recentOrphanStats.tokens || 0);
if (!sections.length) {
if (!sections.length) {
return { promptText: "", injectionLogText: "", injectionStats };
}
@@ -848,4 +835,4 @@ export async function buildVectorPromptText(excludeLastAi = false, hooks = {}) {
}
return { text: finalText, logText: (recallResult.logText || "") + (injectionLogText || "") };
}
}

100
modules/story-summary/vector/recall.js
View File

@@ -296,19 +296,34 @@ function buildEntityLexicon(store, allEvents) {
.slice(0, 5000);
}
function extractEntities(text, lexicon) {
const t = normalize(text);
if (!t || !lexicon?.length) return [];
/**
* 从分段消息中提取实体，继承消息权重
* @param {string[]} segments
* @param {number[]} weights
* @param {string[]} lexicon
* @returns {Map<string, number>}
*/
function extractEntitiesWithWeights(segments, weights, lexicon) {
const entityWeights = new Map();
const sorted = [...lexicon].sort((a, b) => b.length - a.length);
const hits = [];
for (const e of sorted) {
if (t.includes(e)) hits.push(e);
if (hits.length >= 20) break;
if (!segments?.length || !lexicon?.length) return entityWeights;
for (let i = 0; i < segments.length; i++) {
const text = normalize(segments[i]);
const weight = weights?.[i] || 0;
for (const entity of lexicon) {
if (text.includes(entity)) {
const existing = entityWeights.get(entity) || 0;
if (weight > existing) {
entityWeights.set(entity, weight);
}
}
}
}
return hits;
}
return entityWeights;
}
// ═══════════════════════════════════════════════════════════════════════════
// MMR
// ═══════════════════════════════════════════════════════════════════════════
@@ -457,7 +472,7 @@ async function searchChunks(queryVector, vectorConfig, l0FloorBonus = new Map(),
// L2 Events 检索（RRF 混合 + MMR 后置）
// ═══════════════════════════════════════════════════════════════════════════
async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus = new Map()) {
async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus = new Map()) {
const { chatId } = getContext();
if (!chatId || !queryVector?.length) return [];
@@ -475,11 +490,14 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
// 文本路检索
const textRanked = searchEventsByText(queryTextForSearch, CONFIG.TEXT_SEARCH_LIMIT);
const textGapInfo = textRanked._gapInfo || null;
// ═══════════════════════════════════════════════════════════════════════
// 向量路检索（只保留 L0 加权）
// ═══════════════════════════════════════════════════════════════════════
const ENTITY_BONUS_FACTOR = 0.10;
const scored = (allEvents || []).map((event, idx) => {
const v = vectorMap.get(event.id);
const sim = v ? cosineSimilarity(queryVector, v) : 0;
@@ -497,6 +515,17 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
}
}
const participants = (event.participants || []).map(p => normalize(p));
let maxEntityWeight = 0;
for (const p of participants) {
const w = queryEntityWeights.get(p) || 0;
if (w > maxEntityWeight) {
maxEntityWeight = w;
}
}
const entityBonus = ENTITY_BONUS_FACTOR * maxEntityWeight;
bonus += entityBonus;
return {
_id: event.id,
_idx: idx,
@@ -504,9 +533,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
similarity: sim,
finalScore: sim + bonus,
vector: v,
_entityBonus: entityBonus,
};
});
const entityBonusById = new Map(scored.map(s => [s._id, s._entityBonus]));
const preFilterDistribution = {
total: scored.length,
'0.85+': scored.filter(s => s.finalScore >= 0.85).length,
@@ -518,7 +550,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
threshold: CONFIG.MIN_SIMILARITY_EVENT,
};
// 向量路：纯相似度排序（不在这里做 MMR）
const candidates = scored
.filter(s => s.finalScore >= CONFIG.MIN_SIMILARITY_EVENT)
.sort((a, b) => b.finalScore - a.finalScore)
@@ -530,15 +561,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
vector: s.vector,
}));
// RRF 融合
const eventById = new Map(allEvents.map(e => [e.id, e]));
const fused = fuseEventsByRRF(vectorRanked, textRanked, eventById);
// 向量非空时过滤纯 TEXT
const hasVector = vectorRanked.length > 0;
const filtered = hasVector ? fused.filter(x => x.type !== 'TEXT') : fused;
// MMR 放在融合后：对最终候选集去重
const mmrInput = filtered.slice(0, CONFIG.CANDIDATE_EVENTS).map(x => ({
...x,
_id: x.id,
@@ -551,7 +579,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
c => c.vector || null,
c => c.rrf
);
// 构造结果
const results = mmrOutput.map(x => ({
event: x.event,
@@ -559,6 +586,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
_recallType: x.type === 'HYBRID' ? 'DIRECT' : 'SIMILAR',
_recallReason: x.type,
_rrfDetail: { vRank: x.vRank, tRank: x.tRank, rrf: x.rrf },
_entityBonus: entityBonusById.get(x.event?.id) || 0,
}));
// 统计信息附加到第一条结果
@@ -571,6 +599,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
vectorOnlyCount: fused.filter(x => x.type === 'VECTOR').length,
textOnlyFiltered: fused.filter(x => x.type === 'TEXT').length,
};
results[0]._textGapInfo = textGapInfo;
}
return results;
@@ -587,10 +616,11 @@ function formatRecallLog({
chunkResults,
eventResults,
allEvents,
queryEntities,
queryEntityWeights = new Map(),
causalEvents = [],
chunkPreFilterStats = null,
l0Results = [],
textGapInfo = null,
}) {
const lines = [
'\u2554' + '\u2550'.repeat(62) + '\u2557',
@@ -621,7 +651,18 @@ function formatRecallLog({
lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510');
lines.push('\u2502 【提取实体】 \u2502');
lines.push('\u2514' + '\u2500'.repeat(61) + '\u2518');
lines.push(` ${queryEntities?.length ? queryEntities.join('、') : '(无)'}`);
if (queryEntityWeights?.size) {
const sorted = Array.from(queryEntityWeights.entries())
.sort((a, b) => b[1] - a[1])
.slice(0, 8);
const formatted = sorted
.map(([e, w]) => `${e}(${(w * 100).toFixed(0)}%)`)
.join(' | ');
lines.push(` ${formatted}`);
} else {
lines.push(' (无)');
}
lines.push('');
lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510');
@@ -642,7 +683,7 @@ function formatRecallLog({
lines.push(' L1 原文片段:');
if (chunkPreFilterStats) {
const dist = chunkPreFilterStats.distribution || {};
lines.push(` \u5168\u91cf: ${chunkPreFilterStats.total} \u6761 | \u901a\u8fc7\u9608\u503c(\u8fdc\u671f\u2265${chunkPreFilterStats.thresholdRemote}, \u5f85\u6574\u7406\u2265${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} \u6761 | \u6700\u7ec8: ${chunkResults.length} \u6761`);
lines.push(` 全量: ${chunkPreFilterStats.total} 条 | 通过阈值(远期≥${chunkPreFilterStats.thresholdRemote}, 待整理≥${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} 条 | 最终: ${chunkResults.length} 条`);
lines.push(` 匹配度: 0.8+: ${dist['0.8+'] || 0} | 0.7-0.8: ${dist['0.7-0.8'] || 0} | 0.6-0.7: ${dist['0.6-0.7'] || 0}`);
} else {
lines.push(` 选入: ${chunkResults.length} 条`);
@@ -656,6 +697,18 @@ function formatRecallLog({
lines.push(` 总事件: ${allEvents.length} 条 | 最终: ${eventResults.length} 条`);
lines.push(` 向量路: ${rrfStats.vectorCount || 0} 条 | 文本路: ${rrfStats.textCount || 0} 条`);
lines.push(` HYBRID: ${rrfStats.hybridCount || 0} 条 | 纯 VECTOR: ${rrfStats.vectorOnlyCount || 0} 条 | 纯 TEXT (已过滤): ${rrfStats.textOnlyFiltered || 0} 条`);
const entityBoostedEvents = eventResults.filter(e => e._entityBonus > 0).length;
lines.push(` 实体加分事件: ${entityBoostedEvents} 条`);
if (textGapInfo) {
lines.push('');
lines.push(' 文本检索 (BM25 动态 top-K):');
lines.push(` 命中: ${textGapInfo.total} 条 | 返回: ${textGapInfo.returned} 条 (覆盖 ${textGapInfo.coverage} 总分)`);
if (textGapInfo.scoreRange) {
const s = textGapInfo.scoreRange;
lines.push(` 分数: Top=${s.top} | 截断=${s.cutoff} | P50=${s.p50} | Last=${s.last}`);
}
}
// Causal
if (causalEvents.length) {
@@ -702,7 +755,8 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
}
const lexicon = buildEntityLexicon(store, allEvents);
const queryEntities = extractEntities(segments.join('\n'), lexicon);
const queryEntityWeights = extractEntitiesWithWeights(segments, weights, lexicon);
const queryEntities = Array.from(queryEntityWeights.keys());
// 构建文本查询串：最后一条消息 + 实体 + 关键词
const lastSeg = segments[segments.length - 1] || '';
@@ -727,10 +781,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
const [chunkResults, eventResults] = await Promise.all([
searchChunks(queryVector, vectorConfig, l0FloorBonus, lastSummarizedFloor),
searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus),
searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus),
]);
const chunkPreFilterStats = chunkResults._preFilterStats || null;
const textGapInfo = eventResults[0]?._textGapInfo || null;
const mergedChunks = mergeAndSparsify(l0VirtualChunks, chunkResults, CONFIG.FLOOR_MAX_CHUNKS);
@@ -764,10 +819,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
chunkResults: mergedChunks,
eventResults,
allEvents,
queryEntities,
queryEntityWeights,
causalEvents: causalEventsTruncated,
chunkPreFilterStats,
l0Results,
textGapInfo,
});
console.group('%c[Recall]', 'color: #7c3aed; font-weight: bold');

146
modules/story-summary/vector/text-search.js
View File

@@ -1,37 +1,70 @@
// ═══════════════════════════════════════════════════════════════════════════
// Text Search - L2 事件文本检索（MiniSearch）
// 与向量检索互补，通过 RRF 融合
// ═══════════════════════════════════════════════════════════════════════════
// text-search.js - 最终版
import MiniSearch from '../../../libs/minisearch.mjs';
const STOP_WORDS = new Set([
'的', '了', '是', '在', '和', '与', '或', '但', '而', '却',
'这', '那', '他', '她', '它', '我', '你', '们', '着', '过',
'把', '被', '给', '让', '向', '就', '都', '也', '还', '又',
'很', '太', '更', '最', '只', '才', '已', '正', '会', '能',
'要', '可', '得', '地', '之', '所', '以', '为', '于', '有',
'不', '去', '来', '上', '下', '里', '说', '看', '吧', '呢',
'啊', '吗', '呀', '哦', '嗯', '么',
'の', 'に', 'は', 'を', 'が', 'と', 'で', 'へ', 'や', 'か',
'も', 'な', 'よ', 'ね', 'わ', 'です', 'ます', 'した', 'ない',
'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been',
'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
'to', 'of', 'in', 'on', 'at', 'for', 'with', 'by', 'from',
'and', 'or', 'but', 'if', 'that', 'this', 'it', 'its',
'i', 'you', 'he', 'she', 'we', 'they', 'my', 'your', 'his',
]);
function tokenize(text) {
const s = String(text || '').toLowerCase().trim();
if (!s) return [];
const tokens = new Set();
// CJK Bigram + Trigram
const cjk = s.match(/[\u4e00-\u9fff\u3400-\u4dbf]+/g) || [];
for (const seg of cjk) {
const chars = [...seg].filter(c => !STOP_WORDS.has(c));
for (let i = 0; i < chars.length - 1; i++) {
tokens.add(chars[i] + chars[i + 1]);
}
for (let i = 0; i < chars.length - 2; i++) {
tokens.add(chars[i] + chars[i + 1] + chars[i + 2]);
}
}
// 日语假名
const kana = s.match(/[\u3040-\u309f\u30a0-\u30ff]{2,}/g) || [];
for (const k of kana) {
if (!STOP_WORDS.has(k)) tokens.add(k);
}
// 英文
const en = s.match(/[a-z]{2,}/g) || [];
for (const w of en) {
if (!STOP_WORDS.has(w)) tokens.add(w);
}
return [...tokens];
}
let idx = null;
let lastRevision = null;
/**
* 中文逐字 + 英数字串分词
*/
function tokenize(text) {
return String(text || '').match(/[\u4e00-\u9fff]|[a-zA-Z0-9]+/g) || [];
}
/**
* 去掉 summary 末尾的楼层标记
*/
function stripFloorTag(s) {
return String(s || '').replace(/\s*\(#\d+(?:-\d+)?\)\s*$/, '').trim();
}
/**
* 构建/更新事件文本索引
*/
export function ensureEventTextIndex(events, revision) {
if (!events?.length) {
idx = null;
lastRevision = null;
return;
}
if (idx && revision === lastRevision) return;
try {
@@ -39,6 +72,7 @@ export function ensureEventTextIndex(events, revision) {
fields: ['title', 'summary', 'participants'],
storeFields: ['id'],
tokenize,
searchOptions: { tokenize },
});
idx.addAll(events.map(e => ({
@@ -52,33 +86,87 @@ export function ensureEventTextIndex(events, revision) {
} catch (e) {
console.error('[text-search] Index build failed:', e);
idx = null;
lastRevision = null;
}
}
/**
* 文本检索事件
* BM25 检索，返回 top-K 候选给 RRF
*
* 设计原则：
* - 不做分数过滤（BM25 分数跨查询不可比）
* - 不做匹配数过滤（bigram 让一个词产生多个 token）
* - 只做 top-K（BM25 排序本身有区分度）
* - 质量过滤交给 RRF 后的 hasVector 过滤
*/
/**
* 动态 top-K：累积分数占比法
*
* 原理：BM25 分数服从幂律分布，少数高分条目贡献大部分总分
* 取累积分数达到阈值的最小 K
*
* 参考：帕累托法则（80/20 法则）在信息检索中的应用
*/
function dynamicTopK(scores, coverage = 0.90, minK = 15, maxK = 80) {
if (!scores.length) return 0;
const total = scores.reduce((a, b) => a + b, 0);
if (total <= 0) return Math.min(minK, scores.length);
let cumulative = 0;
for (let i = 0; i < scores.length; i++) {
cumulative += scores[i];
if (cumulative / total >= coverage) {
return Math.max(minK, Math.min(maxK, i + 1));
}
}
return Math.min(maxK, scores.length);
}
export function searchEventsByText(queryText, limit = 80) {
if (!idx || !queryText?.trim()) return [];
try {
const res = idx.search(queryText, {
limit,
boost: { title: 2, participants: 1.5, summary: 1 },
fuzzy: 0.2,
prefix: true,
const results = idx.search(queryText, {
boost: { title: 4, participants: 2, summary: 1 },
fuzzy: false,
prefix: false,
});
return res.map((r, i) => ({ id: r.id, textRank: i + 1 }));
if (!results.length) return [];
const scores = results.map(r => r.score);
const k = dynamicTopK(scores, 0.90, 15, limit);
const output = results.slice(0, k).map((r, i) => ({
id: r.id,
textRank: i + 1,
score: r.score,
}));
const total = scores.reduce((a, b) => a + b, 0);
const kCumulative = scores.slice(0, k).reduce((a, b) => a + b, 0);
output._gapInfo = {
total: results.length,
returned: k,
coverage: ((kCumulative / total) * 100).toFixed(1) + '%',
scoreRange: {
top: scores[0]?.toFixed(1),
cutoff: scores[k - 1]?.toFixed(1),
p50: scores[Math.floor(scores.length / 2)]?.toFixed(1),
last: scores[scores.length - 1]?.toFixed(1),
},
};
return output;
} catch (e) {
console.error('[text-search] Search failed:', e);
return [];
}
}
/**
* 清理索引
*/
export function clearEventTextIndex() {
idx = null;
lastRevision = null;