feat(knowledge): 语义检索打通——切片、混合打分、只搜已发布

之前知识只能"存"，不能"找"。这一版把链路补上：切片 → 向量 → 打分 → 带来源返回。 - chunk 表 + 迁移 0006：正文按标题归段切、代码块整块保留、超长按句切、相邻片重叠 80 字；RLS 与 GRANT 与其它租户表一致 - 向量提供方做成协议：内置 hash（字符二元组，零依赖可复现，作为基线）与 OpenAI 兼容端点两种，换实现不动业务代码 - 检索是混合打分：向量 0.6 + pg_trgm 0.4。专有编号、单号这类必须原样命中的词靠词法兜，换个说法表达的同一件事靠向量兜 - 三条纪律：只搜 published（过滤写在 SQL 里）、低于阈值宁可不返回、每条命中都带来源标题与 item_id - 发布时自动建索引，改完正文用 POST /knowledge/{id}/reindex 重建 - Web 知识页加检索框与命中卡片；新增 e2e/retrieval.mjs 6 项为什么不是 pgvector：本机 Docker 起不来、fnsvr-01 没有该扩展且明确不动。先用 real[] + SQL 余弦把链路跑通，将来换列类型与索引即可，接口不变。后端 79 passed，e2e smoke/knowledge/richtext/retrieval 全绿。
2026-10-29 04:28:09 +08:00
parent d509f0c335
commit d079564d6c
16 changed files with 1126 additions and 22 deletions
@@ -0,0 +1,127 @@
+"""把 Markdown 正文切成适合检索的小片段（M3）。
+
+切片的目的是"喂给检索"，不是"给人读"，所以三条规则优先于其它考虑：
+
+1. **不切坏结构**：代码块整块保留，表格不拆行，标题跟着它下面的段落走
+   （单独一句"结论"离开上下文就没意义）。
+2. **片内自洽**：每片尽量包含一个完整意思，超长才按句号切。
+3. **相邻片有重叠**：跨段的话被切断时，前后片都能读到完整句子，命中后引用才不乱。
+
+不引入分词器：中文另起 `zhparser` 要装扩展，而片段足够短时字符级检索已经够用。
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
+FENCE_RE = re.compile(r"^\s*```")
+SENTENCE_RE = re.compile(r"[^。！？!?；;\n]+[。！？!?；;]?")
+
+
+@dataclass(frozen=True)
+class ChunkDraft:
+    heading: str | None
+    content: str
+
+
+def _split_long(text: str, max_chars: int) -> list[str]:
+    """把超长段落按句子切开；单句仍然超长就硬切。"""
+    pieces: list[str] = []
+    buffer = ""
+    for sentence in SENTENCE_RE.findall(text):
+        if not sentence.strip():
+            continue
+        if len(buffer) + len(sentence) <= max_chars:
+            buffer += sentence
+            continue
+        if buffer.strip():
+            pieces.append(buffer.strip())
+        if len(sentence) <= max_chars:
+            buffer = sentence
+        else:
+            for start in range(0, len(sentence), max_chars):
+                pieces.append(sentence[start : start + max_chars].strip())
+            buffer = ""
+    if buffer.strip():
+        pieces.append(buffer.strip())
+    return [p for p in pieces if p]
+
+
+def split_markdown(
+    text: str,
+    *,
+    max_chars: int = 600,
+    overlap: int = 80,
+) -> list[ChunkDraft]:
+    """返回按顺序排列的切片。
+
+    `overlap` 是相邻片之间共享的尾部字数；不想要重叠就传 0。
+    """
+    if not text or not text.strip():
+        return []
+
+    normalized = text.replace("\r\n", "\n").replace("\r", "\n")
+    blocks: list[tuple[str | None, str]] = []
+    heading: str | None = None
+    paragraph: list[str] = []
+    in_fence = False
+
+    def flush() -> None:
+        if paragraph:
+            body = "\n".join(paragraph).strip()
+            if body:
+                blocks.append((heading, body))
+            paragraph.clear()
+
+    for line in normalized.split("\n"):
+        if FENCE_RE.match(line):
+            in_fence = not in_fence
+            paragraph.append(line)
+            continue
+        if not in_fence:
+            match = HEADING_RE.match(line)
+            if match:
+                flush()
+                heading = match.group(2).strip()
+                continue
+            if not line.strip():
+                flush()
+                continue
+        paragraph.append(line)
+    flush()
+
+    chunks: list[ChunkDraft] = []
+    buffer = ""
+    buffer_heading: str | None = None
+
+    def flush_buffer() -> None:
+        nonlocal buffer, buffer_heading
+        if buffer.strip():
+            chunks.append(ChunkDraft(heading=buffer_heading, content=buffer.strip()))
+        buffer = ""
+
+    for block_heading, body in blocks:
+        parts = _split_long(body, max_chars) if len(body) > max_chars else [body]
+        for part in parts:
+            candidate = f"{buffer}\n\n{part}".strip() if buffer else part
+            if len(candidate) <= max_chars:
+                if not buffer:
+                    buffer_heading = block_heading
+                buffer = candidate
+                continue
+            flush_buffer()
+            # 新片开头带上上一片的尾巴，避免"半句话"被切断后两边都读不通
+            tail = ""
+            if overlap > 0 and chunks:
+                tail = chunks[-1].content[-overlap:]
+                if "\n" in tail:
+                    tail = tail.split("\n", 1)[-1]
+            buffer_heading = block_heading
+            buffer = f"{tail}\n\n{part}".strip() if tail else part
+            if len(buffer) > max_chars:
+                # 带尾巴后仍然超长：放弃尾巴，保证不超限
+                buffer = part
+    flush_buffer()
+    return chunks