Update recall entity weighting and prompt sections

2026-02-02 14:02:12 +08:00
parent d8849c5e8b
commit d3f772073f
3 changed files with 229 additions and 98 deletions
--- a/modules/story-summary/vector/recall.js
+++ b/modules/story-summary/vector/recall.js
@@ -296,19 +296,34 @@ function buildEntityLexicon(store, allEvents) {
        .slice(0, 5000);
 }

-function extractEntities(text, lexicon) {
-    const t = normalize(text);
-    if (!t || !lexicon?.length) return [];
+/**
+ * 从分段消息中提取实体，继承消息权重
+ * @param {string[]} segments
+ * @param {number[]} weights
+ * @param {string[]} lexicon
+ * @returns {Map<string, number>}
+ */
+function extractEntitiesWithWeights(segments, weights, lexicon) {
+    const entityWeights = new Map();

-    const sorted = [...lexicon].sort((a, b) => b.length - a.length);
-    const hits = [];
-    for (const e of sorted) {
-        if (t.includes(e)) hits.push(e);
-        if (hits.length >= 20) break;
+    if (!segments?.length || !lexicon?.length) return entityWeights;
+
+    for (let i = 0; i < segments.length; i++) {
+        const text = normalize(segments[i]);
+        const weight = weights?.[i] || 0;
+
+        for (const entity of lexicon) {
+            if (text.includes(entity)) {
+                const existing = entityWeights.get(entity) || 0;
+                if (weight > existing) {
+                    entityWeights.set(entity, weight);
+                }
+            }
+        }
    }
-    return hits;
-}

+    return entityWeights;
+}
 // ═══════════════════════════════════════════════════════════════════════════
 // MMR
 // ═══════════════════════════════════════════════════════════════════════════
@@ -457,7 +472,7 @@ async function searchChunks(queryVector, vectorConfig, l0FloorBonus = new Map(),
 // L2 Events 检索（RRF 混合 + MMR 后置）
 // ═══════════════════════════════════════════════════════════════════════════

-async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus = new Map()) {
+async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus = new Map()) {
    const { chatId } = getContext();
    if (!chatId || !queryVector?.length) return [];

@@ -475,11 +490,14 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo

    // 文本路检索
    const textRanked = searchEventsByText(queryTextForSearch, CONFIG.TEXT_SEARCH_LIMIT);
+    const textGapInfo = textRanked._gapInfo || null;

    // ═══════════════════════════════════════════════════════════════════════
    // 向量路检索（只保留 L0 加权）
    // ═══════════════════════════════════════════════════════════════════════

+    const ENTITY_BONUS_FACTOR = 0.10;
+
    const scored = (allEvents || []).map((event, idx) => {
        const v = vectorMap.get(event.id);
        const sim = v ? cosineSimilarity(queryVector, v) : 0;
@@ -497,6 +515,17 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
            }
        }

+        const participants = (event.participants || []).map(p => normalize(p));
+        let maxEntityWeight = 0;
+        for (const p of participants) {
+            const w = queryEntityWeights.get(p) || 0;
+            if (w > maxEntityWeight) {
+                maxEntityWeight = w;
+            }
+        }
+        const entityBonus = ENTITY_BONUS_FACTOR * maxEntityWeight;
+        bonus += entityBonus;
+
        return {
            _id: event.id,
            _idx: idx,
@@ -504,9 +533,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
            similarity: sim,
            finalScore: sim + bonus,
            vector: v,
+            _entityBonus: entityBonus,
        };
    });

+    const entityBonusById = new Map(scored.map(s => [s._id, s._entityBonus]));
+
    const preFilterDistribution = {
        total: scored.length,
        '0.85+': scored.filter(s => s.finalScore >= 0.85).length,
@@ -518,7 +550,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
        threshold: CONFIG.MIN_SIMILARITY_EVENT,
    };

-    // 向量路：纯相似度排序（不在这里做 MMR）
    const candidates = scored
        .filter(s => s.finalScore >= CONFIG.MIN_SIMILARITY_EVENT)
        .sort((a, b) => b.finalScore - a.finalScore)
@@ -530,15 +561,12 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
        vector: s.vector,
    }));

-    // RRF 融合
    const eventById = new Map(allEvents.map(e => [e.id, e]));
    const fused = fuseEventsByRRF(vectorRanked, textRanked, eventById);

-    // 向量非空时过滤纯 TEXT
    const hasVector = vectorRanked.length > 0;
    const filtered = hasVector ? fused.filter(x => x.type !== 'TEXT') : fused;

-    // MMR 放在融合后：对最终候选集去重
    const mmrInput = filtered.slice(0, CONFIG.CANDIDATE_EVENTS).map(x => ({
        ...x,
        _id: x.id,
@@ -551,7 +579,6 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
        c => c.vector || null,
        c => c.rrf
    );
-
    // 构造结果
    const results = mmrOutput.map(x => ({
        event: x.event,
@@ -559,6 +586,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
        _recallType: x.type === 'HYBRID' ? 'DIRECT' : 'SIMILAR',
        _recallReason: x.type,
        _rrfDetail: { vRank: x.vRank, tRank: x.tRank, rrf: x.rrf },
+        _entityBonus: entityBonusById.get(x.event?.id) || 0,
    }));

    // 统计信息附加到第一条结果
@@ -571,6 +599,7 @@ async function searchEvents(queryVector, queryTextForSearch, allEvents, vectorCo
            vectorOnlyCount: fused.filter(x => x.type === 'VECTOR').length,
            textOnlyFiltered: fused.filter(x => x.type === 'TEXT').length,
        };
+        results[0]._textGapInfo = textGapInfo;
    }

    return results;
@@ -587,10 +616,11 @@ function formatRecallLog({
    chunkResults,
    eventResults,
    allEvents,
-    queryEntities,
+    queryEntityWeights = new Map(),
    causalEvents = [],
    chunkPreFilterStats = null,
    l0Results = [],
+    textGapInfo = null,
 }) {
    const lines = [
        '\u2554' + '\u2550'.repeat(62) + '\u2557',
@@ -621,7 +651,18 @@ function formatRecallLog({
    lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510');
    lines.push('\u2502 【提取实体】                                                 \u2502');
    lines.push('\u2514' + '\u2500'.repeat(61) + '\u2518');
-    lines.push(`  ${queryEntities?.length ? queryEntities.join('、') : '(无)'}`);
+
+    if (queryEntityWeights?.size) {
+        const sorted = Array.from(queryEntityWeights.entries())
+            .sort((a, b) => b[1] - a[1])
+            .slice(0, 8);
+        const formatted = sorted
+            .map(([e, w]) => `${e}(${(w * 100).toFixed(0)}%)`)
+            .join(' | ');
+        lines.push(`  ${formatted}`);
+    } else {
+        lines.push('  (无)');
+    }

    lines.push('');
    lines.push('\u250c' + '\u2500'.repeat(61) + '\u2510');
@@ -642,7 +683,7 @@ function formatRecallLog({
    lines.push('  L1 原文片段:');
    if (chunkPreFilterStats) {
        const dist = chunkPreFilterStats.distribution || {};
-        lines.push(`    \u5168\u91cf: ${chunkPreFilterStats.total} \u6761 | \u901a\u8fc7\u9608\u503c(\u8fdc\u671f\u2265${chunkPreFilterStats.thresholdRemote}, \u5f85\u6574\u7406\u2265${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} \u6761 | \u6700\u7ec8: ${chunkResults.length} \u6761`);
+        lines.push(`    全量: ${chunkPreFilterStats.total} 条 | 通过阈值(远期≥${chunkPreFilterStats.thresholdRemote}, 待整理≥${chunkPreFilterStats.thresholdRecent}): ${chunkPreFilterStats.passThreshold} 条 | 最终: ${chunkResults.length} 条`);
        lines.push(`    匹配度: 0.8+: ${dist['0.8+'] || 0} | 0.7-0.8: ${dist['0.7-0.8'] || 0} | 0.6-0.7: ${dist['0.6-0.7'] || 0}`);
    } else {
        lines.push(`    选入: ${chunkResults.length} 条`);
@@ -656,6 +697,18 @@ function formatRecallLog({
    lines.push(`    总事件: ${allEvents.length} 条 | 最终: ${eventResults.length} 条`);
    lines.push(`    向量路: ${rrfStats.vectorCount || 0} 条 | 文本路: ${rrfStats.textCount || 0} 条`);
    lines.push(`    HYBRID: ${rrfStats.hybridCount || 0} 条 | 纯 VECTOR: ${rrfStats.vectorOnlyCount || 0} 条 | 纯 TEXT (已过滤): ${rrfStats.textOnlyFiltered || 0} 条`);
+    const entityBoostedEvents = eventResults.filter(e => e._entityBonus > 0).length;
+    lines.push(`    实体加分事件: ${entityBoostedEvents} 条`);
+
+    if (textGapInfo) {
+        lines.push('');
+        lines.push('  文本检索 (BM25 动态 top-K):');
+        lines.push(`    命中: ${textGapInfo.total} 条 | 返回: ${textGapInfo.returned} 条 (覆盖 ${textGapInfo.coverage} 总分)`);
+        if (textGapInfo.scoreRange) {
+            const s = textGapInfo.scoreRange;
+            lines.push(`    分数: Top=${s.top} | 截断=${s.cutoff} | P50=${s.p50} | Last=${s.last}`);
+        }
+    }

    // Causal
    if (causalEvents.length) {
@@ -702,7 +755,8 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
    }

    const lexicon = buildEntityLexicon(store, allEvents);
-    const queryEntities = extractEntities(segments.join('\n'), lexicon);
+    const queryEntityWeights = extractEntitiesWithWeights(segments, weights, lexicon);
+    const queryEntities = Array.from(queryEntityWeights.keys());

    // 构建文本查询串：最后一条消息 + 实体 + 关键词
    const lastSeg = segments[segments.length - 1] || '';
@@ -727,10 +781,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =

    const [chunkResults, eventResults] = await Promise.all([
        searchChunks(queryVector, vectorConfig, l0FloorBonus, lastSummarizedFloor),
-        searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntities, l0FloorBonus),
+        searchEvents(queryVector, queryTextForSearch, allEvents, vectorConfig, store, queryEntityWeights, l0FloorBonus),
    ]);

    const chunkPreFilterStats = chunkResults._preFilterStats || null;
+    const textGapInfo = eventResults[0]?._textGapInfo || null;

    const mergedChunks = mergeAndSparsify(l0VirtualChunks, chunkResults, CONFIG.FLOOR_MAX_CHUNKS);

@@ -764,10 +819,11 @@ export async function recallMemory(queryText, allEvents, vectorConfig, options =
        chunkResults: mergedChunks,
        eventResults,
        allEvents,
-        queryEntities,
+        queryEntityWeights,
        causalEvents: causalEventsTruncated,
        chunkPreFilterStats,
        l0Results,
+        textGapInfo,
    });

    console.group('%c[Recall]', 'color: #7c3aed; font-weight: bold');