From d079564d6ce0dd88263c1e2ebd1e69db29d04a26 Mon Sep 17 00:00:00 2001 From: dadao Date: Thu, 29 Oct 2026 04:28:09 +0800 Subject: [PATCH] =?UTF-8?q?feat(knowledge):=20=E8=AF=AD=E4=B9=89=E6=A3=80?= =?UTF-8?q?=E7=B4=A2=E6=89=93=E9=80=9A=E2=80=94=E2=80=94=E5=88=87=E7=89=87?= =?UTF-8?q?=E3=80=81=E6=B7=B7=E5=90=88=E6=89=93=E5=88=86=E3=80=81=E5=8F=AA?= =?UTF-8?q?=E6=90=9C=E5=B7=B2=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 之前知识只能"存"，不能"找"。这一版把链路补上：切片 → 向量 → 打分 → 带来源返回。 - chunk 表 + 迁移 0006：正文按标题归段切、代码块整块保留、超长按句切、 相邻片重叠 80 字；RLS 与 GRANT 与其它租户表一致 - 向量提供方做成协议：内置 hash（字符二元组，零依赖可复现，作为基线） 与 OpenAI 兼容端点两种，换实现不动业务代码 - 检索是混合打分：向量 0.6 + pg_trgm 0.4。专有编号、单号这类必须原样命中的词 靠词法兜，换个说法表达的同一件事靠向量兜 - 三条纪律：只搜 published（过滤写在 SQL 里）、低于阈值宁可不返回、 每条命中都带来源标题与 item_id - 发布时自动建索引，改完正文用 POST /knowledge/{id}/reindex 重建 - Web 知识页加检索框与命中卡片；新增 e2e/retrieval.mjs 6 项 为什么不是 pgvector：本机 Docker 起不来、fnsvr-01 没有该扩展且明确不动。 先用 real[] + SQL 余弦把链路跑通，将来换列类型与索引即可，接口不变。 后端 79 passed，e2e smoke/knowledge/richtext/retrieval 全绿。 --- 2.Design/API.md | 40 ++++- 2.Design/ROADMAP-M0-M1.md | 27 ++- README.md | 4 +- backend/alembic/versions/0006_chunks.py | 102 ++++++++++++ backend/app/api/knowledge.py | 34 ++++ backend/app/config.py | 3 + backend/app/domain/chunking.py | 127 +++++++++++++++ backend/app/models_knowledge.py | 37 ++++- backend/app/services/embedding.py | 131 +++++++++++++++ backend/app/services/knowledge_index.py | 208 ++++++++++++++++++++++++ backend/app/services/signing.py | 4 +- backend/tests/test_attachments.py | 7 +- backend/tests/test_retrieval.py | 200 +++++++++++++++++++++++ web/e2e/retrieval.mjs | 120 ++++++++++++++ web/src/lib/types.ts | 13 ++ web/src/pages/Knowledge.tsx | 91 ++++++++++- 16 files changed, 1126 insertions(+), 22 deletions(-) create mode 100644 backend/alembic/versions/0006_chunks.py create mode 100644 backend/app/domain/chunking.py create mode 100644 backend/app/services/embedding.py create mode 100644 backend/app/services/knowledge_index.py create mode 100644 backend/tests/test_retrieval.py create mode 100644 web/e2e/retrieval.mjs diff --git a/2.Design/API.md b/2.Design/API.md index 6ca6d2f..2c2969c 100644 --- a/2.Design/API.md +++ b/2.Design/API.md @@ -377,7 +377,45 @@ Query：`doc_type`（默认 `postmortem`）。来源可以是任务或工单； **工单验收通过时，后端会自动调用这条逻辑**（`generated_by=rule`，`doc_type=postmortem`， 正文开头固定一句"这份草稿由系统根据来源自动整理，请先核对再发布"），所以用户不需要记得点。 -### 5.7 知识的错误码 +### 5.7 检索 `GET /knowledge/search` + +Query： +| 参数 | 说明 | +|---|---| +| `q` | 必填，1–200 字 | +| `limit` | 1–30，默认 10 | +| `mode` | `hybrid`（默认）/ `vector` / `keyword`，后两个用于对比效果 | + +返回 `{query, mode, items[]}`，每项： + +| 字段 | 说明 | +|---|---| +| `item_id` / `title` | 来源知识，前端"打开原文"用 | +| `doc_type` / `heading` | 类型与所在小节 | +| `idx` / `content` | 第几片、切片正文 | +| `score` | 加权总分，低于 `SEARCH_MIN_SCORE`（0.06）的结果**不返回** | +| `vector_score` / `lexical_score` | 向量分与 trigram 词法分，便于排查"为什么没搜到" | + +三条约定： + +1. **只检索 `published`**。草稿是还没被人认可的话，进检索等于把猜测当结论引用出去——过滤写在 SQL 里，调用方改不了。 +2. **结果必须带来源**，能点回原文。这是 D3（知识必须有来源）的延伸：不给"出处不存在"的答案。 +3. **宁可返回空，也不凑数**。检索失败比误导便宜。 + +发布（`5.5` 的 `publish`）时会自动重建索引，用户发布完立刻能搜到； +但改完正文不会自动重建，需要调用下面这条。 + +### `POST /knowledge/{item_id}/reindex` — 重建切片 + +返回 `{item_id, chunk_count, searchable}`。`searchable=false` 表示内容已建好切片但还不是 `published`， +搜不到——这不是错误。 + +实现：切片存 `chunk` 表（`content` + `embedding real[]`），相似度用 SQL 函数 `app_cosine()` 计算。 +本机没有可用的 pgvector 实例（Docker 起不来，fnsvr-01 明确不动），所以先用 `real[]` 把链路跑通； +将来换 pgvector 只改列类型与索引，接口不变。换 embedding 模型（`EMBEDDING_PROVIDER`）后 +**必须整体重建**，否则新旧向量混在一张表里，分数没法比较。 + +### 5.8 知识的错误码 | 码 | HTTP | 含义 | |---|---|---| diff --git a/2.Design/ROADMAP-M0-M1.md b/2.Design/ROADMAP-M0-M1.md index 4293ae9..a8a07bb 100644 --- a/2.Design/ROADMAP-M0-M1.md +++ b/2.Design/ROADMAP-M0-M1.md @@ -272,21 +272,32 @@ | 3.5 | 手动沉淀：任务/工单详情里"沉淀成知识" | 0.5 | [x] 有 e2e | | 3.6 | 手记：没有来源的知识（唯一允许 `source_kind=manual` 的入口） | 0.5 | [x] 手记页 + 列表 | | 3.7 | 检索：标题/正文/摘要子串 + 类型/状态筛选 + 游标分页 | 1.0 | [x] trigram 索引在 0002 已建 | -| 3.8 | 切片与向量索引（`chunk` 表、pgvector HNSW） | 2.0 | [ ] 见下方说明 | -| 3.9 | 安卓知识列表 | 1.0 | [ ] 与工单一起延后到编译链就绪 | -| 3.10 | 文档与 e2e | 1.0 | [x] `web/e2e/knowledge.mjs` 7 项断言绿 | +| 3.8 | 切片与向量索引（`chunk` 表、pgvector HNSW） | 2.0 | [x] **用 `real[]` + SQL 余弦先跑通**（见下方说明） | +| 3.8.1 | 切片：按标题归段、代码块整块保留、超长按句切、相邻片重叠 80 字 | 0.5 | [x] `app/domain/chunking.py` | +| 3.8.2 | 向量：`hash`（字符二元组）与 `openai` 两种提供方，换实现不动业务 | 0.5 | [x] `app/services/embedding.py` | +| 3.8.3 | 混合检索：向量 0.6 + trigram 0.4，低于阈值不返回，只认 `published` | 1.0 | [x] 5 个用例 + `e2e/retrieval.mjs` 6 项 | +| 3.8.4 | 索引生命周期：发布自动建、改正文需 `reindex`、换模型整体重建 | 0.5 | [x] `POST /knowledge/{id}/reindex` | +| 3.9 | 安卓知识列表 | 1.0 | [x] 只读列表 + 详情动作（编辑正文走安卓编辑器） | +| 3.10 | 文档与 e2e | 1.0 | [x] `knowledge.mjs` 7 项 + `retrieval.mjs` 6 项 | ### M3 完成定义（DOD） - [x] 一件事办完 → 自动出现一份草稿 → 核对 → 发布，全程不用手动复制粘贴 - [x] 每条知识都能追到来源（除手记），点开能看到来源事项 - [x] 非管理员发布被拒；跨租户看不到别人的知识 -- [ ] 向量切片与语义检索（需要独立 pgvector 实例真正跑起来） +- [x] 切片与语义检索跑通（`real[]` + SQL 余弦；pgvector 仍待独立实例） - [ ] 真实用两周：新同事靠知识库自己解决过一次问题（需要真实团队） -**关于 3.8 的说明**：本机 Docker 起不来、fnsvr-01 上也没有 pgvector 扩展， -所以 `deploy/docker-compose.vector.yml` 与 `VECTOR_DATABASE_URL` 只有配置，**没有跑过**。 -按"写完不算完成"的规矩，它在真跑起来之前保持未勾。切片表设计要点已写进 -`2.Design/PRODUCT-PLAN.md` 的 D8，落地时不要改 `EMBEDDING_DIM`（改了要重算全量）。 +**关于 3.8 的说明（2026-10-29 已落地，但与计划不同）**：计划里写的是 pgvector + HNSW， +实际本机 Docker 起不来、fnsvr-01 上也没有 pgvector 扩展且用户明确要求别动那台机器， +所以先用 `chunk` 表（`embedding real[]`）+ SQL 函数 `app_cosine()` 把整条链路跑通： +切片 → 向量（内置 hash 提供方）→ 混合打分（向量 0.6 + trigram 0.4）→ 阈值过滤 → 带来源返回。 +`deploy/docker-compose.vector.yml` 与 `VECTOR_DATABASE_URL` 保留，等向量实例真起来之后再切过去， +换 pgvector 只改列类型与索引，应用层接口不变。 + +两条必须记住的约束： +1. **内置 hash 向量不是语义模型**，只是"零依赖、可复现"的基线；上线前换 `EMBEDDING_PROVIDER=openai` + 并整体重建（`reindex_workspace`），否则新旧向量混表、分数不可比。 +2. **检索只认 `published`**，草稿永远搜不到——这条写在 SQL 的 WHERE 里，不靠调用方自觉。 --- diff --git a/README.md b/README.md index 840c656..0082afc 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,8 @@ | 知识：工单验收自动生成草稿、核对后发布、来源可追、手记 | ✅ 后端 + Web + 安卓已通，e2e 7 项全绿 | | 界面：两端统一设计系统（纸白 / 墨绿 / 陶土橙，语义色、字号阶、组件库） | ✅ 已重做，见 [`2.Design/ROADMAP-M0-M1.md`](2.Design/ROADMAP-M0-M1.md) 第 4 节 | | 富文本：备注 / 工单说明 / 知识正文支持图文混排与附件 | ✅ Web 用 Vditor 编辑、安卓用 Markwon 预览编辑（e2e 15 项全绿）；接口见 [`2.Design/API.md`](2.Design/API.md) 第 6 节 | -| 后端测试 | ✅ 74 passed，ruff 干净 | -| 知识切片与语义检索（pgvector） | ⬜ 配置已就绪（`deploy/docker-compose.vector.yml`），本机 Docker 起不来，未跑过 | +| 知识切片与语义检索（混合打分、只搜已发布、带来源） | ✅ 已跑通：`real[]` + SQL 余弦（pgvector 待独立实例），`pytest` 5 个用例 + `e2e/retrieval.mjs` 6 项 | +| 后端测试 | ✅ 79 passed，ruff 干净 | | LLM 能力（结构化录入、问答带引用、人物画像） | ⬜ M4 | ## 快速开始 diff --git a/backend/alembic/versions/0006_chunks.py b/backend/alembic/versions/0006_chunks.py new file mode 100644 index 0000000..4efdfc9 --- /dev/null +++ b/backend/alembic/versions/0006_chunks.py @@ -0,0 +1,102 @@ +"""知识切片与检索（M3） + +Revision ID: 0006 +Revises: 0005 +Create Date: 2026-10-29 + +建 `chunk` 表：知识正文按段切开，每片一行，带向量数组。 + +两个刻意的取舍： + +1. **向量存 `real[]`，不依赖 pgvector**。本机拿不到带 pgvector 的实例（Docker 起不来， + fnsvr-01 明确不动），但检索链路不能一直停在纸面上。`real[]` + 一条 SQL 余弦函数 + 在小数据量下够用；将来换上 pgvector 只需改列类型、加 HNSW 索引，应用层接口不变。 +2. **检索范围只能是 `published`**。草稿是"还没被人认可的话"，进检索等于让 AI 把 + 猜测当结论引用出去——这条过滤写在查询里，不靠调用方自觉。 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +from alembic import op + +revision: str = "0006" +down_revision: str = "0005" +branch_labels: Sequence[str] | None = None +depends_on: Sequence[str] | None = None + +# 余弦相似度。写成 SQL 函数而不是在 Python 里算，是为了让过滤与排序留在数据库里做—— +# 把整表向量捞回应用算相似度，数据一多就会拖垮 API。 +COSINE_SQL = """ +CREATE OR REPLACE FUNCTION app_cosine(a real[], b real[]) RETURNS double precision +LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE AS $fn$ +DECLARE + n int; + dot double precision := 0; + na double precision := 0; + nb double precision := 0; +BEGIN + n := LEAST(array_length(a, 1), array_length(b, 1)); + IF n IS NULL THEN + RETURN NULL; + END IF; + FOR i IN 1..n LOOP + dot := dot + a[i] * b[i]; + na := na + a[i] * a[i]; + nb := nb + b[i] * b[i]; + END LOOP; + IF na = 0 OR nb = 0 THEN + RETURN 0; + END IF; + RETURN dot / sqrt(na * nb); +END +$fn$; +""" + +CREATE_SQL = """ +CREATE TABLE chunk ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + workspace_id uuid NOT NULL, + item_id uuid NOT NULL REFERENCES item(id) ON DELETE CASCADE, + idx integer NOT NULL, + heading varchar(200), + content text NOT NULL, + char_count integer NOT NULL DEFAULT 0, + embedding real[], + embed_model varchar(64), + version integer NOT NULL DEFAULT 1, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT uq_chunk_item_idx UNIQUE (item_id, idx) +) +""" + +# asyncpg 的执行器不接受多语句，一条 `execute` 只能带一句 SQL（0001 踩过同样的坑） +INDEX_STATEMENTS = ( + "CREATE INDEX ix_chunk_ws_item ON chunk (workspace_id, item_id)", + "COMMENT ON TABLE chunk IS '知识正文切片，检索的最小单位'", +) + +POLICY_SQL = ( + "ALTER TABLE chunk ENABLE ROW LEVEL SECURITY", + "CREATE POLICY chunk_isolation ON chunk " + "USING (workspace_id = app_ws_id()) WITH CHECK (workspace_id = app_ws_id())", + "GRANT SELECT, INSERT, UPDATE, DELETE ON chunk TO dadao_app", +) + + +def upgrade() -> None: + op.execute(COSINE_SQL) + op.execute(CREATE_SQL) + for statement in INDEX_STATEMENTS: + op.execute(statement) + for statement in POLICY_SQL: + op.execute(statement) + + +def downgrade() -> None: + op.execute("DROP POLICY IF EXISTS chunk_isolation ON chunk") + op.execute("DROP TABLE IF EXISTS chunk") + # 函数被别的东西引用时先别删；本轮只有切片在用，留一条注释说明为什么还留着。 + # op.execute("DROP FUNCTION IF EXISTS app_cosine(real[], real[])") diff --git a/backend/app/api/knowledge.py b/backend/app/api/knowledge.py index 36e7f03..04f16f9 100644 --- a/backend/app/api/knowledge.py +++ b/backend/app/api/knowledge.py @@ -33,6 +33,7 @@ from ..models_task import Activity, Item from ..schemas import clamp_limit, decode_cursor, encode_cursor, ok from ..serializers import item_out, tags_of from ..services.knowledge import create_draft_from_source, find_doc_by_source +from ..services.knowledge_index import reindex_document, search_knowledge from .attachments import count_for_items, list_for_item router = APIRouter(prefix="/workspaces/{ws_id}", tags=["知识库"]) @@ -203,6 +204,20 @@ async def list_knowledge( return ok({"items": items, "next_cursor": next_cursor, "has_more": has_more}) +# 注意路由顺序：`/knowledge/search` 必须排在 `/knowledge/{item_id}` 之前， +# 否则 "search" 会被当成 uuid 解析，返回 422。 +@router.get("/knowledge/search", summary="检索已发布知识") +async def search( + ctx: MemberDep, + session: SessionDep, + q: Annotated[str, Query(min_length=1, max_length=200, description="检索词")], + limit: Annotated[int, Query(ge=1, le=30)] = 10, + mode: Annotated[str, Query(pattern="^(hybrid|vector|keyword)$")] = "hybrid", +) -> dict: + hits = await search_knowledge(session, ctx.ws_id, q, limit=limit, mode=mode) + return ok({"query": q, "mode": mode, "items": hits, "has_more": False}) + + @router.get("/knowledge/{item_id}", summary="知识详情") async def get_knowledge(item_id: uuid.UUID, ctx: MemberDep, session: SessionDep) -> dict: item, doc = await _load_doc(session, ctx.ws_id, item_id) @@ -396,9 +411,28 @@ async def change_status( payload={"to": to_status}, ) await session.flush() + if body.action == "publish": + # 发布后立刻建索引：检索只认 published，不在这里建就要等定时任务， + # 而"刚发布却搜不到"是最容易被当成 bug 的体验。 + await reindex_document(session, ws_id, item.id) return ok(await _knowledge_out(session, ws_id, item, doc)) +@router.post("/knowledge/{item_id}/reindex", summary="重建这份知识的检索切片") +async def reindex(item_id: uuid.UUID, ctx: MemberDep, session: SessionDep) -> dict: + # 改完正文不重建，检索命中的还是旧句子——留一个能手动触发的入口 + _item, doc = await _load_doc(session, ctx.ws_id, item_id) + count = await reindex_document(session, ctx.ws_id, item_id) + await session.flush() + return ok( + { + "item_id": item_id, + "chunk_count": count, + "searchable": doc.status_hint == "published", + } + ) + + # ── 从任务/工单沉淀 ───────────────────────────────────────────────────── @router.post("/items/{item_id}/knowledge", summary="把这件事沉淀成知识草稿") async def promote_to_knowledge( diff --git a/backend/app/config.py b/backend/app/config.py index 55c84f6..e7b58bb 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -89,6 +89,9 @@ class Settings(BaseSettings): search_vector_weight: float = 0.6 search_lexical_weight: float = 0.4 + # 低于这个分数就不返回：宁可"没找到"，也不要塞一条不相干的让人误信。 + search_min_score: float = 0.06 + # ── 知识库向量库（独立实例，M3 起用）─────────────────── # 刻意与业务库分开放：fnsvr-01 上的 PG 没有 pgvector，也不该为了装扩展去动它； # 向量检索吃内存，和任务库挤在同一实例会互相拖累。 diff --git a/backend/app/domain/chunking.py b/backend/app/domain/chunking.py new file mode 100644 index 0000000..d51855b --- /dev/null +++ b/backend/app/domain/chunking.py @@ -0,0 +1,127 @@ +"""把 Markdown 正文切成适合检索的小片段（M3）。 + +切片的目的是"喂给检索"，不是"给人读"，所以三条规则优先于其它考虑： + +1. **不切坏结构**：代码块整块保留，表格不拆行，标题跟着它下面的段落走 + （单独一句"结论"离开上下文就没意义）。 +2. **片内自洽**：每片尽量包含一个完整意思，超长才按句号切。 +3. **相邻片有重叠**：跨段的话被切断时，前后片都能读到完整句子，命中后引用才不乱。 + +不引入分词器：中文另起 `zhparser` 要装扩展，而片段足够短时字符级检索已经够用。 +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +FENCE_RE = re.compile(r"^\s*```") +SENTENCE_RE = re.compile(r"[^。！？!?；;\n]+[。！？!?；;]?") + + +@dataclass(frozen=True) +class ChunkDraft: + heading: str | None + content: str + + +def _split_long(text: str, max_chars: int) -> list[str]: + """把超长段落按句子切开；单句仍然超长就硬切。""" + pieces: list[str] = [] + buffer = "" + for sentence in SENTENCE_RE.findall(text): + if not sentence.strip(): + continue + if len(buffer) + len(sentence) <= max_chars: + buffer += sentence + continue + if buffer.strip(): + pieces.append(buffer.strip()) + if len(sentence) <= max_chars: + buffer = sentence + else: + for start in range(0, len(sentence), max_chars): + pieces.append(sentence[start : start + max_chars].strip()) + buffer = "" + if buffer.strip(): + pieces.append(buffer.strip()) + return [p for p in pieces if p] + + +def split_markdown( + text: str, + *, + max_chars: int = 600, + overlap: int = 80, +) -> list[ChunkDraft]: + """返回按顺序排列的切片。 + + `overlap` 是相邻片之间共享的尾部字数；不想要重叠就传 0。 + """ + if not text or not text.strip(): + return [] + + normalized = text.replace("\r\n", "\n").replace("\r", "\n") + blocks: list[tuple[str | None, str]] = [] + heading: str | None = None + paragraph: list[str] = [] + in_fence = False + + def flush() -> None: + if paragraph: + body = "\n".join(paragraph).strip() + if body: + blocks.append((heading, body)) + paragraph.clear() + + for line in normalized.split("\n"): + if FENCE_RE.match(line): + in_fence = not in_fence + paragraph.append(line) + continue + if not in_fence: + match = HEADING_RE.match(line) + if match: + flush() + heading = match.group(2).strip() + continue + if not line.strip(): + flush() + continue + paragraph.append(line) + flush() + + chunks: list[ChunkDraft] = [] + buffer = "" + buffer_heading: str | None = None + + def flush_buffer() -> None: + nonlocal buffer, buffer_heading + if buffer.strip(): + chunks.append(ChunkDraft(heading=buffer_heading, content=buffer.strip())) + buffer = "" + + for block_heading, body in blocks: + parts = _split_long(body, max_chars) if len(body) > max_chars else [body] + for part in parts: + candidate = f"{buffer}\n\n{part}".strip() if buffer else part + if len(candidate) <= max_chars: + if not buffer: + buffer_heading = block_heading + buffer = candidate + continue + flush_buffer() + # 新片开头带上上一片的尾巴，避免"半句话"被切断后两边都读不通 + tail = "" + if overlap > 0 and chunks: + tail = chunks[-1].content[-overlap:] + if "\n" in tail: + tail = tail.split("\n", 1)[-1] + buffer_heading = block_heading + buffer = f"{tail}\n\n{part}".strip() if tail else part + if len(buffer) > max_chars: + # 带尾巴后仍然超长：放弃尾巴，保证不超限 + buffer = part + flush_buffer() + return chunks diff --git a/backend/app/models_knowledge.py b/backend/app/models_knowledge.py index 3e11572..9b54dda 100644 --- a/backend/app/models_knowledge.py +++ b/backend/app/models_knowledge.py @@ -19,15 +19,18 @@ from datetime import datetime from sqlalchemy import ( CheckConstraint, DateTime, + Float, ForeignKey, Index, String, Text, + UniqueConstraint, ) +from sqlalchemy.dialects.postgresql import ARRAY from sqlalchemy.dialects.postgresql import UUID as PgUUID from sqlalchemy.orm import Mapped, mapped_column -from .models import Base, Timestamps +from .models import Base, Timestamps, new_uuid # 知识状态机（与任务/工单共用 item.status 一列，取值集合不同） KNOWLEDGE_STATUSES: tuple[str, ...] = ("draft", "in_review", "published", "archived") @@ -76,4 +79,34 @@ class KnowledgeDoc(Base, Timestamps): chunk_count: Mapped[int] = mapped_column(nullable=False, default=0) -KNOWLEDGE_TENANT_TABLES: tuple[str, ...] = ("knowledge_doc",) +class Chunk(Base): + """知识正文的切片。 + + 为什么切片与文档同库：检索要按工作区过滤，跨库做权限校验就得把两边数据捞到应用里， + 既不安全也不快。向量本身是普通数组，换 pgvector 时只换列类型与索引。 + """ + + __tablename__ = "chunk" + __table_args__ = ( + UniqueConstraint("item_id", "idx", name="uq_chunk_item_idx"), + Index("ix_chunk_ws_item", "workspace_id", "item_id"), + {"comment": "知识正文切片，检索的最小单位"}, + ) + + id: Mapped[uuid.UUID] = mapped_column(PgUUID(as_uuid=True), primary_key=True, default=new_uuid) + workspace_id: Mapped[uuid.UUID] = mapped_column(PgUUID(as_uuid=True), nullable=False) + item_id: Mapped[uuid.UUID] = mapped_column( + PgUUID(as_uuid=True), ForeignKey("item.id", ondelete="CASCADE"), nullable=False + ) + # 片内顺序，用于把命中片段按原顺序还原上下文 + idx: Mapped[int] = mapped_column(nullable=False) + heading: Mapped[str | None] = mapped_column(String(200)) + content: Mapped[str] = mapped_column(Text, nullable=False) + char_count: Mapped[int] = mapped_column(nullable=False, default=0) + # 向量。hash 提供方是 256 维，OpenAI 兼容端点通常是 1024/1536 维， + # 所以不固定维度——换模型时老切片要么重算，要么维度不匹配（检索会拒绝并提示重建）。 + embedding: Mapped[list[float] | None] = mapped_column(ARRAY(Float)) + embed_model: Mapped[str | None] = mapped_column(String(64)) + + +KNOWLEDGE_TENANT_TABLES: tuple[str, ...] = ("knowledge_doc", "chunk") diff --git a/backend/app/services/embedding.py b/backend/app/services/embedding.py new file mode 100644 index 0000000..9c609e8 --- /dev/null +++ b/backend/app/services/embedding.py @@ -0,0 +1,131 @@ +"""文本向量化（M3）。 + +为什么要抽象一层：今天用内置的哈希向量，明天换成 OpenAI 兼容的 `/embeddings`， +后天可能换成自建模型。检索代码只依赖 `embed() -> list[float]`，换实现不动业务。 + +内置 `HashEmbedding` 的诚实说明：它不是"语义模型"，是**字符 n-gram 的哈希词袋**。 +它能把同词、近义写法（"发票"/"票据"这种）之外的东西区分开，抓不到真正的语义相似。 +之所以先上它：本机没有可用的向量库与模型服务，检索链路不能一直不落地； +而且它零依赖、可复现（同样的文本永远同样的向量），适合做测试基线。 +真上线前应当把 `EMBEDDING_PROVIDER` 换成 `openai`，并把已发布知识整体重建一次。 +""" + +from __future__ import annotations + +import hashlib +import math +import re +from collections.abc import Sequence +from typing import Protocol + +import httpx + +from ..config import Settings, get_settings + +CJK_RE = re.compile(r"[\u4e00-\u9fff]") +WORD_RE = re.compile(r"[a-z0-9]+") + + +class EmbeddingProvider(Protocol): + """向量提供方。`name` 会写进切片，换模型后能看出哪些是旧向量。""" + + name: str + + async def embed(self, texts: Sequence[str]) -> list[list[float]]: ... + + +def _tokens(text: str) -> list[str]: + """中文按字二元组、英文按词。 + + 中文不用分词器：单字太碎（"发""票"都对不上"发票"），二元组能拿到大多数 + 有意义的搭配，又不需要维护词典。 + """ + lowered = text.lower() + tokens = WORD_RE.findall(lowered) + chars = [c for c in lowered if CJK_RE.match(c)] + tokens.extend(chars[i] + chars[i + 1] for i in range(len(chars) - 1)) + tokens.extend(chars) + return tokens + + +class HashEmbedding: + """字符 n-gram 的带符号哈希词袋，L2 归一化。""" + + def __init__(self, dim: int = 256) -> None: + self.dim = dim + self.name = f"hash-{dim}" + + async def embed(self, texts: Sequence[str]) -> list[list[float]]: + return [self._one(text) for text in texts] + + def _one(self, text: str) -> list[float]: + vector = [0.0] * self.dim + counts: dict[str, int] = {} + for token in _tokens(text): + counts[token] = counts.get(token, 0) + 1 + for token, count in counts.items(): + digest = hashlib.blake2b(token.encode("utf-8"), digest_size=8).digest() + raw = int.from_bytes(digest, "big") + index = raw % self.dim + # 用最高位决定正负：无符号哈希会让不相干的词在向量里越加越像 + sign = 1.0 if (raw >> 63) & 1 else -1.0 + vector[index] += sign * (1.0 + math.log(count)) + norm = math.sqrt(sum(v * v for v in vector)) + if norm == 0: + return vector + return [v / norm for v in vector] + + +class OpenAIEmbedding: + """任何 OpenAI 兼容的 `/embeddings` 端点（BYOK）。""" + + def __init__(self, *, base_url: str, api_key: str, model: str) -> None: + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self.model = model + self.name = f"openai:{model}" + + async def embed(self, texts: Sequence[str]) -> list[list[float]]: + if not texts: + return [] + async with httpx.AsyncClient(timeout=30.0) as client: + response = await client.post( + f"{self.base_url}/embeddings", + headers={"Authorization": f"Bearer {self.api_key}"}, + json={"model": self.model, "input": list(texts)}, + ) + response.raise_for_status() + payload = response.json() + # 按 index 排序：并发端点的返回顺序没有保证 + rows = sorted(payload["data"], key=lambda row: row["index"]) + return [row["embedding"] for row in rows] + + +_provider: EmbeddingProvider | None = None + + +def get_provider(settings: Settings | None = None) -> EmbeddingProvider: + """进程内单例：哈希实现无状态，OpenAI 实现也不需要每次新建连接池。""" + global _provider + if _provider is not None: + return _provider + settings = settings or get_settings() + if settings.embedding_provider == "openai": + if not settings.embedding_api_key: + raise RuntimeError( + "EMBEDDING_PROVIDER=openai 但没有配置 EMBEDDING_API_KEY；" + "要么补上密钥，要么改回 hash" + ) + _provider = OpenAIEmbedding( + base_url=settings.embedding_base_url, + api_key=settings.embedding_api_key, + model=settings.embedding_model, + ) + else: + _provider = HashEmbedding() + return _provider + + +def reset_provider_for_tests() -> None: + global _provider + _provider = None diff --git a/backend/app/services/knowledge_index.py b/backend/app/services/knowledge_index.py new file mode 100644 index 0000000..5fd7239 --- /dev/null +++ b/backend/app/services/knowledge_index.py @@ -0,0 +1,208 @@ +"""知识的切片索引与检索（M3）。 + +两条硬规则： + +1. **只有 `published` 的知识能被检索到**。草稿是"还没被人认可的话"， + 让它进检索等于把猜测当结论引用出去。过滤写在 SQL 里，不靠调用方自觉。 +2. **检索必须带来源**。每个命中都返回 `item_id` 与标题，用户能点回去看原文； + 这和 D3（知识必须有来源）是同一条原则的延伸——不给"出处不存在"的答案。 + +评分是向量与词法的加权和： +- 向量负责"说的是同一件事但换了词"； +- 词法（pg_trgm 的 trigram 相似度）负责"专有名词、单号、人名"这类必须原样命中的词， + 而哈希向量对这类词并不敏感。 +两者都不足时宁可返回空，也不硬凑一条不相关的结果——检索失败比误导便宜。 +""" + +from __future__ import annotations + +import uuid +from typing import Any + +from sqlalchemy import delete, select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from ..config import get_settings +from ..domain.chunking import split_markdown +from ..errors import AppError, ErrorCode +from ..models_knowledge import Chunk, KnowledgeDoc +from ..models_task import Item +from .embedding import get_provider + +SEARCH_SQL = text( + """ + SELECT c.item_id, + i.title, + k.doc_type, + c.idx, + c.heading, + c.content, + coalesce(app_cosine(c.embedding, CAST(:qvec AS real[])), 0) AS vec_score, + coalesce(similarity(c.content, :q), 0) AS lex_score + FROM chunk c + JOIN item i ON i.id = c.item_id AND i.workspace_id = c.workspace_id + JOIN knowledge_doc k ON k.item_id = c.item_id AND k.workspace_id = c.workspace_id + WHERE c.workspace_id = :ws + AND k.status_hint = 'published' + ORDER BY ( + coalesce(app_cosine(c.embedding, CAST(:qvec AS real[])), 0) * :vw + + coalesce(similarity(c.content, :q), 0) * :lw + ) DESC, c.item_id, c.idx + LIMIT :limit + """ +) + + +def _vector_param(values: list[float]) -> list[float]: + """把向量交给驱动去编码。 + + 一开始我拼成 `'{0.1,0.2}'` 字符串再 `CAST(... AS real[])`，asyncpg 会因为 CAST + 已经告诉它"这是数组"而拒绝字符串（`a sized iterable container expected`）。 + 直接传 list 最省事；非有限值替换成 0，免得 NaN 进 SQL 变成错误输入。 + """ + return [value if value == value and abs(value) != float("inf") else 0.0 for value in values] + + +async def reindex_document( + session: AsyncSession, + ws_id: uuid.UUID, + item_id: uuid.UUID, +) -> int: + """重建一份知识的切片，返回切片数。 + + 全量重建而不是增量：知识正文改动通常牵动上下几段，增量对齐的复杂度 + 换不来多少收益（一份知识撑死几十片）。 + """ + item = ( + await session.execute(select(Item).where(Item.workspace_id == ws_id, Item.id == item_id)) + ).scalar_one_or_none() + if item is None: + raise AppError(ErrorCode.ITEM_NOT_FOUND, "这件事不存在") + + doc = ( + await session.execute( + select(KnowledgeDoc).where( + KnowledgeDoc.workspace_id == ws_id, + KnowledgeDoc.item_id == item_id, + ) + ) + ).scalar_one_or_none() + if doc is None: + raise AppError(ErrorCode.KNOWLEDGE_NOT_FOUND, "这份知识不存在") + + settings = get_settings() + source = "\n\n".join(part for part in (item.title, item.body_md or "") if part) + drafts = split_markdown( + source, + max_chars=settings.chunk_max_chars, + overlap=settings.chunk_overlap_chars, + ) + + provider = get_provider(settings) + vectors = await provider.embed([draft.content for draft in drafts]) if drafts else [] + + await session.execute( + delete(Chunk).where(Chunk.workspace_id == ws_id, Chunk.item_id == item_id) + ) + for index, (draft, vector) in enumerate(zip(drafts, vectors, strict=True)): + session.add( + Chunk( + workspace_id=ws_id, + item_id=item_id, + idx=index, + heading=draft.heading, + content=draft.content, + char_count=len(draft.content), + embedding=vector, + embed_model=provider.name, + ) + ) + doc.chunk_count = len(drafts) + await session.flush() + return len(drafts) + + +async def search_knowledge( + session: AsyncSession, + ws_id: uuid.UUID, + query: str, + *, + limit: int = 10, + mode: str = "hybrid", +) -> list[dict[str, Any]]: + """检索已发布知识，返回按相关度排序的切片。 + + `mode`: hybrid（默认）/ vector / keyword —— 前两者主要用于对比效果。 + """ + cleaned = (query or "").strip() + if not cleaned: + return [] + + settings = get_settings() + provider = get_provider(settings) + query_vector = (await provider.embed([cleaned]))[0] + + if mode == "vector": + vector_weight, lexical_weight = 1.0, 0.0 + elif mode == "keyword": + vector_weight, lexical_weight = 0.0, 1.0 + else: + vector_weight, lexical_weight = ( + settings.search_vector_weight, + settings.search_lexical_weight, + ) + + rows = ( + await session.execute( + SEARCH_SQL, + { + "qvec": _vector_param(query_vector), + "q": cleaned, + "ws": str(ws_id), + "vw": vector_weight, + "lw": lexical_weight, + "limit": limit, + }, + ) + ).mappings() + + hits: list[dict[str, Any]] = [] + for row in rows: + vector_score = float(row["vec_score"] or 0.0) + lexical_score = float(row["lex_score"] or 0.0) + hits.append( + { + "item_id": row["item_id"], + "title": row["title"], + "doc_type": row["doc_type"], + "idx": row["idx"], + "heading": row["heading"], + "content": row["content"], + "score": round(vector_score * vector_weight + lexical_score * lexical_weight, 4), + "vector_score": round(vector_score, 4), + "lexical_score": round(lexical_score, 4), + } + ) + hits = [hit for hit in hits if hit["score"] >= settings.search_min_score] + return hits + + +async def reindex_workspace(session: AsyncSession, ws_id: uuid.UUID) -> int: + """把一个工作区里已发布的知识全部重建，返回处理的知识份数。 + + 换 embedding 模型之后必须整体跑一次，否则新旧向量混在一张表里， + 相似度分数没法比较。 + """ + item_ids = ( + await session.execute( + select(KnowledgeDoc.item_id).where( + KnowledgeDoc.workspace_id == ws_id, + KnowledgeDoc.status_hint == "published", + ) + ) + ).scalars() + count = 0 + for item_id in item_ids: + await reindex_document(session, ws_id, item_id) + count += 1 + return count diff --git a/backend/app/services/signing.py b/backend/app/services/signing.py index 4623d9e..a3670db 100644 --- a/backend/app/services/signing.py +++ b/backend/app/services/signing.py @@ -37,7 +37,9 @@ def sign(workspace_id: uuid.UUID, attachment_id: uuid.UUID, ttl_seconds: int | N return f"exp={exp}&sig={_digest(workspace_id, attachment_id, exp)}" -def verify(workspace_id: uuid.UUID, attachment_id: uuid.UUID, exp: int | None, sig: str | None) -> bool: +def verify( + workspace_id: uuid.UUID, attachment_id: uuid.UUID, exp: int | None, sig: str | None +) -> bool: if not exp or not sig: return False if exp < int(time.time()): # 过期就是过期，不给宽限 diff --git a/backend/tests/test_attachments.py b/backend/tests/test_attachments.py index bc8c9dd..4145f35 100644 --- a/backend/tests/test_attachments.py +++ b/backend/tests/test_attachments.py @@ -8,13 +8,12 @@ from __future__ import annotations +import uuid from typing import Any import pytest from httpx import AsyncClient -import uuid - from app.config import get_settings from app.services import signing @@ -178,7 +177,7 @@ async def test_raw_url_uses_short_lived_signature(client: AsyncClient) -> None: """图片直链自带短期签名：不需要把登录令牌贴进 URL，也不需要 Authorization 头。""" user, ws, item_id = await _ws_with_item(client) body = b"\x89PNG\r\n\x1a\n signed png" - created = unwrap(await _upload(client, user, ws["id"], item_id, body, "签名图.png")) + unwrap(await _upload(client, user, ws["id"], item_id, body, "签名图.png")) listed = unwrap( await client.get( @@ -225,4 +224,4 @@ async def test_raw_url_signature_is_workspace_scoped(client: AsyncClient) -> Non f"{BASE}/workspaces/{ws_b['id']}/attachments/{created['id']}/raw?{query}" ) assert crossed.status_code == 403 - assert crossed.json()["code"] == 8003 \ No newline at end of file + assert crossed.json()["code"] == 8003 diff --git a/backend/tests/test_retrieval.py b/backend/tests/test_retrieval.py new file mode 100644 index 0000000..baa3ab9 --- /dev/null +++ b/backend/tests/test_retrieval.py @@ -0,0 +1,200 @@ +"""M3 检索测试：切片、索引、跨租户。 + +检索最容易做错的地方不是"搜得准不准"，而是"什么该被搜到、什么绝不能被搜到"： + +1. **草稿不能被检索到**。草稿是还没被人认可的话，让它出现等于把猜测当结论； +2. **刚发布就能搜到**。发布后要立刻建索引，否则用户会以为发布失败； +3. **检索结果必须带来源**，能点回原文； +4. **别的租户搜不到我的知识**，即使关键词一模一样。 +""" + +from __future__ import annotations + +from typing import Any + +from httpx import AsyncClient + +from .conftest import create_workspace, login, unwrap + +V1 = "/api/v1" + + +async def _knowledge( + client: AsyncClient, + user: dict[str, Any], + ws_id: str, + title: str, + body: str, + doc_type: str = "note", +) -> dict[str, Any]: + return unwrap( + await client.post( + f"{V1}/workspaces/{ws_id}/knowledge", + json={"title": title, "body_md": body, "doc_type": doc_type}, + headers=user["headers"], + ) + ) + + +async def _publish( + client: AsyncClient, user: dict[str, Any], ws_id: str, doc: dict[str, Any] +) -> dict[str, Any]: + return unwrap( + await client.post( + f"{V1}/workspaces/{ws_id}/knowledge/{doc['id']}/status", + json={"action": "publish", "version": doc["version"]}, + headers=user["headers"], + ) + ) + + +async def _search( + client: AsyncClient, + user: dict[str, Any], + ws_id: str, + query: str, + mode: str = "hybrid", +) -> list[dict[str, Any]]: + return unwrap( + await client.get( + f"{V1}/workspaces/{ws_id}/knowledge/search", + params={"q": query, "mode": mode}, + headers=user["headers"], + ) + )["items"] + + +async def test_published_is_searchable_draft_is_not(client: AsyncClient) -> None: + owner = await login(client) + ws = await create_workspace(client, owner, "检索测试") + + published = await _knowledge( + client, owner, ws["id"], "报销发票怎么开", "电子发票原件必须一起寄，复印件不算。" + ) + parked = await _knowledge( + client, owner, ws["id"], "供应商账号变更", "变更收款账号需要法人签字并盖章。" + ) + await _publish(client, owner, ws["id"], published) + + hits = await _search(client, owner, ws["id"], "发票") + ids = {hit["item_id"] for hit in hits} + assert published["id"] in ids + assert parked["id"] not in ids, "草稿不该出现在检索结果里" + assert hits[0]["score"] > 0 + assert hits[0]["title"], "检索结果必须带来源标题" + + # 还没验证过的知识，关键词再怎么对也不该冒出来 + draft_hits = await _search(client, owner, ws["id"], "法人签字") + assert all(hit["item_id"] != parked["id"] for hit in draft_hits) + + +async def test_publish_builds_chunks(client: AsyncClient) -> None: + owner = await login(client) + ws = await create_workspace(client, owner, "切片测试") + + body = "\n\n".join( + [ + "## 背景", + "十月的报销单据被财务退回两次，第一次是发票复印件，第二次是缺少行程单。" * 20, + "## 结论", + "报销必须同时提供电子发票原件与行程单，缺一不可。" * 20, + ] + ) + doc = await _knowledge(client, owner, ws["id"], "十月报销被退单复盘", body, "postmortem") + published = await _publish(client, owner, ws["id"], doc) + assert published["knowledge"]["chunk_count"] >= 2, "长正文应该被切成多片" + + detail = unwrap( + await client.get( + f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}", headers=owner["headers"] + ) + ) + assert detail["knowledge"]["chunk_count"] >= 2 + + +async def test_reindex_picks_up_edits(client: AsyncClient) -> None: + owner = await login(client) + ws = await create_workspace(client, owner, "重建索引") + + doc = await _knowledge(client, owner, ws["id"], "值班安排", "周一由张三负责。") + published = await _publish(client, owner, ws["id"], doc) + assert published["knowledge"]["chunk_count"] == 1 + + # 已发布的内容不许直接改（7003）：产品上的路径是「归档 → 恢复成草稿 → 改」 + archived = unwrap( + await client.post( + f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/status", + json={"action": "archive", "version": published["version"]}, + headers=owner["headers"], + ) + ) + draft = unwrap( + await client.post( + f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/status", + json={"action": "restore", "version": archived["version"]}, + headers=owner["headers"], + ) + ) + patched = unwrap( + await client.patch( + f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}", + json={ + "body_md": "周一由张三负责，周三改由李四负责，节假日另行通知。", + "version": draft["version"], + }, + headers=owner["headers"], + ) + ) + # 改完正文但没重建索引，旧切片还在——这就是 reindex 存在的理由 + before = await _search(client, owner, ws["id"], "李四") + assert all("李四" not in hit["content"] for hit in before) + + result = unwrap( + await client.post( + f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/reindex", + headers=owner["headers"], + ) + ) + assert result["chunk_count"] >= 1 + # 仍是草稿，重建了也不该被检索到 + assert result["searchable"] is False + assert await _search(client, owner, ws["id"], "李四") == [] + + await _publish(client, owner, ws["id"], patched) + after = await _search(client, owner, ws["id"], "李四") + assert any("李四" in hit["content"] for hit in after) + + +async def test_search_is_tenant_scoped(client: AsyncClient) -> None: + alice = await login(client) + bob = await login(client) + ws_a = await create_workspace(client, alice, "甲队") + ws_b = await create_workspace(client, bob, "乙队") + + doc = await _knowledge(client, alice, ws_a["id"], "发票规则", "电子发票原件必须一起寄。") + await _publish(client, alice, ws_a["id"], doc) + + assert await _search(client, bob, ws_b["id"], "发票") == [] + # 直接拿别人的工作区搜也不行 + denied = await client.get( + f"{V1}/workspaces/{ws_a['id']}/knowledge/search", + params={"q": "发票"}, + headers=bob["headers"], + ) + assert denied.status_code in (403, 404) + + +async def test_keyword_mode_reports_lexical_score(client: AsyncClient) -> None: + owner = await login(client) + ws = await create_workspace(client, owner, "词法检索") + + doc = await _knowledge( + client, owner, ws["id"], "顺丰单号核对", "单号 SF1234567890 必须与发货记录一致。" + ) + await _publish(client, owner, ws["id"], doc) + + hits = await _search(client, owner, ws["id"], "SF1234567890", mode="keyword") + assert hits, "带专有编号的查询在词法模式下应该命中" + assert hits[0]["lexical_score"] > 0 + # keyword 模式里向量权重是 0：总分只由词法分决定 + assert hits[0]["score"] == hits[0]["lexical_score"] diff --git a/web/e2e/retrieval.mjs b/web/e2e/retrieval.mjs new file mode 100644 index 0000000..3ddd178 --- /dev/null +++ b/web/e2e/retrieval.mjs @@ -0,0 +1,120 @@ +/** + * 检索 e2e：发布的知识能被搜到，草稿搜不到，结果能点回原文。 + * + * 只验证"用户能不能用上"，切片与打分逻辑由后端测试覆盖。 + */ +import { chromium } from "playwright-core"; + +const WEB = process.env.WEB ?? "http://127.0.0.1:5180"; +const API = process.env.API ?? "http://127.0.0.1:8100/api/v1"; +const stamp = String(Date.now()).slice(-7); +const phone = `134${stamp}0`; + +let failures = 0; +function check(name, ok, extra = "") { + console.log(`${ok ? "PASS" : "FAIL"} ${name}${extra ? ` — ${extra}` : ""}`); + if (!ok) failures += 1; +} + +async function call(method, path, body, token) { + const res = await fetch(`${API}${path}`, { + method, + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}), + ...(method === "GET" ? {} : { "idempotency-key": `e2e-${Math.random()}` }), + }, + body: body ? JSON.stringify(body) : undefined, + }); + const text = await res.text(); + const json = text ? JSON.parse(text) : {}; + if (json.code !== 0) throw new Error(`${method} ${path} -> ${json.code} ${json.message}`); + return json.data; +} + +const browser = await chromium.launch({ + executablePath: + process.env.CHROME ?? + `${process.env.LOCALAPPDATA}\\ms-playwright\\chromium-1223\\chrome-win64\\chrome.exe`, +}); +const consoleErrors = []; +const context = await browser.newContext({ viewport: { width: 1280, height: 900 } }); +const page = await context.newPage(); +page.on("console", (m) => { + if (m.type() === "error") consoleErrors.push(m.text()); +}); +page.on("response", (r) => { + if (r.status() >= 400) consoleErrors.push(`${r.status()} ${r.url()}`); +}); + +try { + await call("POST", "/auth/sms/send", { phone }); + const login = await call("POST", "/auth/login", { phone, code: "123456" }); + const token = login.tokens.access_token; + const ws = await call("POST", "/workspaces", { name: `检索验收 ${stamp}` }, token); + + const published = await call( + "POST", + `/workspaces/${ws.id}/knowledge`, + { + title: "快递单号对不上怎么处理", + doc_type: "howto", + body_md: + "## 步骤\n\n先在顺丰后台按手机号反查单号，再和发货记录逐一核对。\n\n如果差超过两天，直接找客服开异常件工单。", + }, + token, + ); + await call( + "POST", + `/workspaces/${ws.id}/knowledge/${published.id}/status`, + { action: "publish", version: published.version }, + token, + ); + + const draft = await call( + "POST", + `/workspaces/${ws.id}/knowledge`, + { title: "还没核对的草稿", doc_type: "note", body_md: "这条写的是异常件的赔付口径，尚未定稿。" }, + token, + ); + + await page.addInitScript( + ([access, refresh]) => { + localStorage.setItem("dadao.access", access); + localStorage.setItem("dadao.refresh", refresh); + }, + [token, login.tokens.refresh_token], + ); + await page.goto(`${WEB}/w/${ws.id}/knowledge`, { waitUntil: "networkidle" }); + + const box = page.getByPlaceholder(/搜标题或正文/); + await box.fill("异常件单号核对"); + await box.press("Enter"); + await page.getByText(/命中 \d+ 段/).waitFor({ timeout: 10_000 }); + const summary = await page.getByText(/命中 \d+ 段/).first().textContent(); + check("检索返回了结果", /命中 [1-9]\d* 段/.test(summary ?? ""), summary ?? ""); + check("命中里带标题", await page.getByText("快递单号对不上怎么处理", { exact: false }).first().isVisible()); + check("草稿不会出现在检索结果里", !(await page.getByText("还没核对的草稿", { exact: false }).count())); + await page.screenshot({ path: "e2e/artifacts/retrieval-hits.png", fullPage: false }); + + await page.getByRole("button", { name: "打开原文", exact: true }).first().click(); + await page.getByText(/已发布 · 版本 \d+/, { exact: false }).waitFor({ timeout: 10_000 }); + check("命中能点回原文", true); + await page.getByRole("button", { name: "关闭", exact: true }).click(); + + await box.fill("量子纠缠"); + await box.press("Enter"); + await page.getByText("没有检索到相关内容", { exact: false }).waitFor({ timeout: 10_000 }); + check("不相干的关键词宁可返回空", true); + + check("浏览器控制台与网络无报错", consoleErrors.length === 0, consoleErrors.slice(0, 3).join(" | ")); +} catch (err) { + failures += 1; + console.log(`FAIL 脚本异常 — ${err.message}`); + await page.screenshot({ path: "e2e/artifacts/retrieval-failure.png" }).catch(() => {}); + console.log((await page.textContent("body"))?.slice(0, 600)); +} finally { + await browser.close(); + console.log(failures === 0 ? "\nALL PASS" : `\n${failures} FAILED`); + process.exit(failures === 0 ? 0 : 1); +} diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index 8190f67..6687bbf 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -157,3 +157,16 @@ export interface Knowledge extends Item { attachment_count?: number; activities?: Activity[]; } + +/** 一条检索命中：切片级结果，带来源，点得回原文。 */ +export interface SearchHit { + item_id: string; + title: string; + doc_type: string; + idx: number; + heading: string | null; + content: string; + score: number; + vector_score: number; + lexical_score: number; +} diff --git a/web/src/pages/Knowledge.tsx b/web/src/pages/Knowledge.tsx index 8207e98..930ce5c 100644 --- a/web/src/pages/Knowledge.tsx +++ b/web/src/pages/Knowledge.tsx @@ -14,7 +14,7 @@ import { StatusPill, } from "../components/ui"; import { ApiError, api } from "../lib/client"; -import type { Attachment, Knowledge, KnowledgeStatus } from "../lib/types"; +import type { Attachment, Knowledge, KnowledgeStatus, SearchHit } from "../lib/types"; const TABS = [ { key: "draft", label: "待核对", statuses: ["draft", "in_review"] }, @@ -63,6 +63,8 @@ export function KnowledgePage() { const [openId, setOpenId] = useState(null); const [adding, setAdding] = useState(false); const [draftTitle, setDraftTitle] = useState(""); + const [hits, setHits] = useState(null); + const [searching, setSearching] = useState(false); const reqId = useRef(0); const load = useCallback( @@ -99,6 +101,32 @@ export function KnowledgePage() { void load(null); }, [load]); + /** + * 语义检索走的是切片接口，和上面的列表筛选不是一回事： + * 列表按标题/正文做子串匹配，检索按向量 + trigram 打分，只覆盖已发布的知识。 + * 所以不把 q 直接接到列表上——两套结果混在一起会让人搞不清命中在哪。 + */ + async function runSearch() { + const keyword = q.trim(); + if (!keyword) { + setHits(null); + return; + } + setSearching(true); + setError(null); + try { + const query = new URLSearchParams({ q: keyword, limit: "12" }); + const page = await api.get<{ items: SearchHit[] }>( + `/workspaces/${wsId}/knowledge/search?${query.toString()}`, + ); + setHits(page.items); + } catch (err) { + setError(err instanceof ApiError ? err.message : "检索失败"); + } finally { + setSearching(false); + } + } + async function addNote() { const title = draftTitle.trim(); if (!title) return; @@ -133,11 +161,21 @@ export function KnowledgePage() { setQ(e.target.value)} + onChange={(e) => { + setQ(e.target.value); + if (!e.target.value.trim()) setHits(null); + }} + onKeyDown={(e) => { + if (e.key === "Enter") void runSearch(); + }} /> + + + {hits.length === 0 ? ( +
+ } + title="没有检索到相关内容" + hint="只检索已发布的知识。换成更具体的说法，或者先把手头的草稿核对发布。" + /> +
+ ) : ( + hits.map((hit) => ( +
+
+ + {DOC_TYPE_LABEL[hit.doc_type] ?? hit.doc_type} + + {hit.heading ? ( + {hit.heading} + ) : null} + + 相关度 {hit.score.toFixed(2)} + +
+

{hit.title}

+

+ {hit.content} +

+
+ +
+
+ )) + )} + + ) : loading && docs.length === 0 ? (