feat(knowledge): 语义检索打通——切片、混合打分、只搜已发布

之前知识只能"存"，不能"找"。这一版把链路补上：切片 → 向量 → 打分 → 带来源返回。 - chunk 表 + 迁移 0006：正文按标题归段切、代码块整块保留、超长按句切、相邻片重叠 80 字；RLS 与 GRANT 与其它租户表一致 - 向量提供方做成协议：内置 hash（字符二元组，零依赖可复现，作为基线）与 OpenAI 兼容端点两种，换实现不动业务代码 - 检索是混合打分：向量 0.6 + pg_trgm 0.4。专有编号、单号这类必须原样命中的词靠词法兜，换个说法表达的同一件事靠向量兜 - 三条纪律：只搜 published（过滤写在 SQL 里）、低于阈值宁可不返回、每条命中都带来源标题与 item_id - 发布时自动建索引，改完正文用 POST /knowledge/{id}/reindex 重建 - Web 知识页加检索框与命中卡片；新增 e2e/retrieval.mjs 6 项为什么不是 pgvector：本机 Docker 起不来、fnsvr-01 没有该扩展且明确不动。先用 real[] + SQL 余弦把链路跑通，将来换列类型与索引即可，接口不变。后端 79 passed，e2e smoke/knowledge/richtext/retrieval 全绿。
2026-10-29 04:28:09 +08:00
parent d509f0c335
commit d079564d6c
16 changed files with 1126 additions and 22 deletions
@@ -0,0 +1,131 @@
+"""文本向量化（M3）。
+
+为什么要抽象一层：今天用内置的哈希向量，明天换成 OpenAI 兼容的 `/embeddings`，
+后天可能换成自建模型。检索代码只依赖 `embed() -> list[float]`，换实现不动业务。
+
+内置 `HashEmbedding` 的诚实说明：它不是"语义模型"，是**字符 n-gram 的哈希词袋**。
+它能把同词、近义写法（"发票"/"票据"这种）之外的东西区分开，抓不到真正的语义相似。
+之所以先上它：本机没有可用的向量库与模型服务，检索链路不能一直不落地；
+而且它零依赖、可复现（同样的文本永远同样的向量），适合做测试基线。
+真上线前应当把 `EMBEDDING_PROVIDER` 换成 `openai`，并把已发布知识整体重建一次。
+"""
+
+from __future__ import annotations
+
+import hashlib
+import math
+import re
+from collections.abc import Sequence
+from typing import Protocol
+
+import httpx
+
+from ..config import Settings, get_settings
+
+CJK_RE = re.compile(r"[\u4e00-\u9fff]")
+WORD_RE = re.compile(r"[a-z0-9]+")
+
+
+class EmbeddingProvider(Protocol):
+    """向量提供方。`name` 会写进切片，换模型后能看出哪些是旧向量。"""
+
+    name: str
+
+    async def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
+
+
+def _tokens(text: str) -> list[str]:
+    """中文按字二元组、英文按词。
+
+    中文不用分词器：单字太碎（"发""票"都对不上"发票"），二元组能拿到大多数
+    有意义的搭配，又不需要维护词典。
+    """
+    lowered = text.lower()
+    tokens = WORD_RE.findall(lowered)
+    chars = [c for c in lowered if CJK_RE.match(c)]
+    tokens.extend(chars[i] + chars[i + 1] for i in range(len(chars) - 1))
+    tokens.extend(chars)
+    return tokens
+
+
+class HashEmbedding:
+    """字符 n-gram 的带符号哈希词袋，L2 归一化。"""
+
+    def __init__(self, dim: int = 256) -> None:
+        self.dim = dim
+        self.name = f"hash-{dim}"
+
+    async def embed(self, texts: Sequence[str]) -> list[list[float]]:
+        return [self._one(text) for text in texts]
+
+    def _one(self, text: str) -> list[float]:
+        vector = [0.0] * self.dim
+        counts: dict[str, int] = {}
+        for token in _tokens(text):
+            counts[token] = counts.get(token, 0) + 1
+        for token, count in counts.items():
+            digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest()
+            raw = int.from_bytes(digest, "big")
+            index = raw % self.dim
+            # 用最高位决定正负：无符号哈希会让不相干的词在向量里越加越像
+            sign = 1.0 if (raw >> 63) & 1 else -1.0
+            vector[index] += sign * (1.0 + math.log(count))
+        norm = math.sqrt(sum(v * v for v in vector))
+        if norm == 0:
+            return vector
+        return [v / norm for v in vector]
+
+
+class OpenAIEmbedding:
+    """任何 OpenAI 兼容的 `/embeddings` 端点（BYOK）。"""
+
+    def __init__(self, *, base_url: str, api_key: str, model: str) -> None:
+        self.base_url = base_url.rstrip("/")
+        self.api_key = api_key
+        self.model = model
+        self.name = f"openai:{model}"
+
+    async def embed(self, texts: Sequence[str]) -> list[list[float]]:
+        if not texts:
+            return []
+        async with httpx.AsyncClient(timeout=30.0) as client:
+            response = await client.post(
+                f"{self.base_url}/embeddings",
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                json={"model": self.model, "input": list(texts)},
+            )
+            response.raise_for_status()
+            payload = response.json()
+        # 按 index 排序：并发端点的返回顺序没有保证
+        rows = sorted(payload["data"], key=lambda row: row["index"])
+        return [row["embedding"] for row in rows]
+
+
+_provider: EmbeddingProvider | None = None
+
+
+def get_provider(settings: Settings | None = None) -> EmbeddingProvider:
+    """进程内单例：哈希实现无状态，OpenAI 实现也不需要每次新建连接池。"""
+    global _provider
+    if _provider is not None:
+        return _provider
+    settings = settings or get_settings()
+    if settings.embedding_provider == "openai":
+        if not settings.embedding_api_key:
+            raise RuntimeError(
+                "EMBEDDING_PROVIDER=openai 但没有配置 EMBEDDING_API_KEY；"
+                "要么补上密钥，要么改回 hash"
+            )
+        _provider = OpenAIEmbedding(
+            base_url=settings.embedding_base_url,
+            api_key=settings.embedding_api_key,
+            model=settings.embedding_model,
+        )
+    else:
+        _provider = HashEmbedding()
+    return _provider
+
+
+def reset_provider_for_tests() -> None:
+    global _provider
+    _provider = None
@@ -0,0 +1,208 @@
+"""知识的切片索引与检索（M3）。
+
+两条硬规则：
+
+1. **只有 `published` 的知识能被检索到**。草稿是"还没被人认可的话"，
+   让它进检索等于把猜测当结论引用出去。过滤写在 SQL 里，不靠调用方自觉。
+2. **检索必须带来源**。每个命中都返回 `item_id` 与标题，用户能点回去看原文；
+   这和 D3（知识必须有来源）是同一条原则的延伸——不给"出处不存在"的答案。
+
+评分是向量与词法的加权和：
+- 向量负责"说的是同一件事但换了词"；
+- 词法（pg_trgm 的 trigram 相似度）负责"专有名词、单号、人名"这类必须原样命中的词，
+  而哈希向量对这类词并不敏感。
+两者都不足时宁可返回空，也不硬凑一条不相关的结果——检索失败比误导便宜。
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy import delete, select, text
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from ..config import get_settings
+from ..domain.chunking import split_markdown
+from ..errors import AppError, ErrorCode
+from ..models_knowledge import Chunk, KnowledgeDoc
+from ..models_task import Item
+from .embedding import get_provider
+
+SEARCH_SQL = text(
+    """
+    SELECT c.item_id,
+           i.title,
+           k.doc_type,
+           c.idx,
+           c.heading,
+           c.content,
+           coalesce(app_cosine(c.embedding, CAST(:qvec AS real[])), 0) AS vec_score,
+           coalesce(similarity(c.content, :q), 0) AS lex_score
+    FROM chunk c
+    JOIN item i ON i.id = c.item_id AND i.workspace_id = c.workspace_id
+    JOIN knowledge_doc k ON k.item_id = c.item_id AND k.workspace_id = c.workspace_id
+    WHERE c.workspace_id = :ws
+      AND k.status_hint = 'published'
+    ORDER BY (
+        coalesce(app_cosine(c.embedding, CAST(:qvec AS real[])), 0) * :vw
+        + coalesce(similarity(c.content, :q), 0) * :lw
+    ) DESC, c.item_id, c.idx
+    LIMIT :limit
+    """
+)
+
+
+def _vector_param(values: list[float]) -> list[float]:
+    """把向量交给驱动去编码。
+
+    一开始我拼成 `'{0.1,0.2}'` 字符串再 `CAST(... AS real[])`，asyncpg 会因为 CAST
+    已经告诉它"这是数组"而拒绝字符串（`a sized iterable container expected`）。
+    直接传 list 最省事；非有限值替换成 0，免得 NaN 进 SQL 变成错误输入。
+    """
+    return [value if value == value and abs(value) != float("inf") else 0.0 for value in values]
+
+
+async def reindex_document(
+    session: AsyncSession,
+    ws_id: uuid.UUID,
+    item_id: uuid.UUID,
+) -> int:
+    """重建一份知识的切片，返回切片数。
+
+    全量重建而不是增量：知识正文改动通常牵动上下几段，增量对齐的复杂度
+    换不来多少收益（一份知识撑死几十片）。
+    """
+    item = (
+        await session.execute(select(Item).where(Item.workspace_id == ws_id, Item.id == item_id))
+    ).scalar_one_or_none()
+    if item is None:
+        raise AppError(ErrorCode.ITEM_NOT_FOUND, "这件事不存在")
+
+    doc = (
+        await session.execute(
+            select(KnowledgeDoc).where(
+                KnowledgeDoc.workspace_id == ws_id,
+                KnowledgeDoc.item_id == item_id,
+            )
+        )
+    ).scalar_one_or_none()
+    if doc is None:
+        raise AppError(ErrorCode.KNOWLEDGE_NOT_FOUND, "这份知识不存在")
+
+    settings = get_settings()
+    source = "\n\n".join(part for part in (item.title, item.body_md or "") if part)
+    drafts = split_markdown(
+        source,
+        max_chars=settings.chunk_max_chars,
+        overlap=settings.chunk_overlap_chars,
+    )
+
+    provider = get_provider(settings)
+    vectors = await provider.embed([draft.content for draft in drafts]) if drafts else []
+
+    await session.execute(
+        delete(Chunk).where(Chunk.workspace_id == ws_id, Chunk.item_id == item_id)
+    )
+    for index, (draft, vector) in enumerate(zip(drafts, vectors, strict=True)):
+        session.add(
+            Chunk(
+                workspace_id=ws_id,
+                item_id=item_id,
+                idx=index,
+                heading=draft.heading,
+                content=draft.content,
+                char_count=len(draft.content),
+                embedding=vector,
+                embed_model=provider.name,
+            )
+        )
+    doc.chunk_count = len(drafts)
+    await session.flush()
+    return len(drafts)
+
+
+async def search_knowledge(
+    session: AsyncSession,
+    ws_id: uuid.UUID,
+    query: str,
+    *,
+    limit: int = 10,
+    mode: str = "hybrid",
+) -> list[dict[str, Any]]:
+    """检索已发布知识，返回按相关度排序的切片。
+
+    `mode`: hybrid（默认）/ vector / keyword —— 前两者主要用于对比效果。
+    """
+    cleaned = (query or "").strip()
+    if not cleaned:
+        return []
+
+    settings = get_settings()
+    provider = get_provider(settings)
+    query_vector = (await provider.embed([cleaned]))[0]
+
+    if mode == "vector":
+        vector_weight, lexical_weight = 1.0, 0.0
+    elif mode == "keyword":
+        vector_weight, lexical_weight = 0.0, 1.0
+    else:
+        vector_weight, lexical_weight = (
+            settings.search_vector_weight,
+            settings.search_lexical_weight,
+        )
+
+    rows = (
+        await session.execute(
+            SEARCH_SQL,
+            {
+                "qvec": _vector_param(query_vector),
+                "q": cleaned,
+                "ws": str(ws_id),
+                "vw": vector_weight,
+                "lw": lexical_weight,
+                "limit": limit,
+            },
+        )
+    ).mappings()
+
+    hits: list[dict[str, Any]] = []
+    for row in rows:
+        vector_score = float(row["vec_score"] or 0.0)
+        lexical_score = float(row["lex_score"] or 0.0)
+        hits.append(
+            {
+                "item_id": row["item_id"],
+                "title": row["title"],
+                "doc_type": row["doc_type"],
+                "idx": row["idx"],
+                "heading": row["heading"],
+                "content": row["content"],
+                "score": round(vector_score * vector_weight + lexical_score * lexical_weight, 4),
+                "vector_score": round(vector_score, 4),
+                "lexical_score": round(lexical_score, 4),
+            }
+        )
+    hits = [hit for hit in hits if hit["score"] >= settings.search_min_score]
+    return hits
+
+
+async def reindex_workspace(session: AsyncSession, ws_id: uuid.UUID) -> int:
+    """把一个工作区里已发布的知识全部重建，返回处理的知识份数。
+
+    换 embedding 模型之后必须整体跑一次，否则新旧向量混在一张表里，
+    相似度分数没法比较。
+    """
+    item_ids = (
+        await session.execute(
+            select(KnowledgeDoc.item_id).where(
+                KnowledgeDoc.workspace_id == ws_id,
+                KnowledgeDoc.status_hint == "published",
+            )
+        )
+    ).scalars()
+    count = 0
+    for item_id in item_ids:
+        await reindex_document(session, ws_id, item_id)
+        count += 1
+    return count
@@ -37,7 +37,9 @@ def sign(workspace_id: uuid.UUID, attachment_id: uuid.UUID, ttl_seconds: int | N
    return f"exp={exp}&sig={_digest(workspace_id, attachment_id, exp)}"


-def verify(workspace_id: uuid.UUID, attachment_id: uuid.UUID, exp: int | None, sig: str | None) -> bool:
+def verify(
+    workspace_id: uuid.UUID, attachment_id: uuid.UUID, exp: int | None, sig: str | None
+) -> bool:
    if not exp or not sig:
        return False
    if exp < int(time.time()):  # 过期就是过期，不给宽限