dadao
d079564d6c
CI / backend (push) Canceled after 0s
之前知识只能"存"，不能"找"。这一版把链路补上：切片 → 向量 → 打分 → 带来源返回。
- chunk 表 + 迁移 0006：正文按标题归段切、代码块整块保留、超长按句切、
相邻片重叠 80 字；RLS 与 GRANT 与其它租户表一致
- 向量提供方做成协议：内置 hash（字符二元组，零依赖可复现，作为基线）
与 OpenAI 兼容端点两种，换实现不动业务代码
- 检索是混合打分：向量 0.6 + pg_trgm 0.4。专有编号、单号这类必须原样命中的词
靠词法兜，换个说法表达的同一件事靠向量兜
- 三条纪律：只搜 published（过滤写在 SQL 里）、低于阈值宁可不返回、
每条命中都带来源标题与 item_id
- 发布时自动建索引，改完正文用 POST /knowledge/{id}/reindex 重建
- Web 知识页加检索框与命中卡片；新增 e2e/retrieval.mjs 6 项
为什么不是 pgvector：本机 Docker 起不来、fnsvr-01 没有该扩展且明确不动。
先用 real[] + SQL 余弦把链路跑通，将来换列类型与索引即可，接口不变。
后端 79 passed，e2e smoke/knowledge/richtext/retrieval 全绿。
201 lines
7.1 KiB
Python
201 lines
7.1 KiB
Python
"""M3 检索测试：切片、索引、跨租户。
|
|
|
|
检索最容易做错的地方不是"搜得准不准"，而是"什么该被搜到、什么绝不能被搜到"：
|
|
|
|
1. **草稿不能被检索到**。草稿是还没被人认可的话，让它出现等于把猜测当结论；
|
|
2. **刚发布就能搜到**。发布后要立刻建索引，否则用户会以为发布失败；
|
|
3. **检索结果必须带来源**，能点回原文；
|
|
4. **别的租户搜不到我的知识**，即使关键词一模一样。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from httpx import AsyncClient
|
|
|
|
from .conftest import create_workspace, login, unwrap
|
|
|
|
V1 = "/api/v1"
|
|
|
|
|
|
async def _knowledge(
|
|
client: AsyncClient,
|
|
user: dict[str, Any],
|
|
ws_id: str,
|
|
title: str,
|
|
body: str,
|
|
doc_type: str = "note",
|
|
) -> dict[str, Any]:
|
|
return unwrap(
|
|
await client.post(
|
|
f"{V1}/workspaces/{ws_id}/knowledge",
|
|
json={"title": title, "body_md": body, "doc_type": doc_type},
|
|
headers=user["headers"],
|
|
)
|
|
)
|
|
|
|
|
|
async def _publish(
|
|
client: AsyncClient, user: dict[str, Any], ws_id: str, doc: dict[str, Any]
|
|
) -> dict[str, Any]:
|
|
return unwrap(
|
|
await client.post(
|
|
f"{V1}/workspaces/{ws_id}/knowledge/{doc['id']}/status",
|
|
json={"action": "publish", "version": doc["version"]},
|
|
headers=user["headers"],
|
|
)
|
|
)
|
|
|
|
|
|
async def _search(
|
|
client: AsyncClient,
|
|
user: dict[str, Any],
|
|
ws_id: str,
|
|
query: str,
|
|
mode: str = "hybrid",
|
|
) -> list[dict[str, Any]]:
|
|
return unwrap(
|
|
await client.get(
|
|
f"{V1}/workspaces/{ws_id}/knowledge/search",
|
|
params={"q": query, "mode": mode},
|
|
headers=user["headers"],
|
|
)
|
|
)["items"]
|
|
|
|
|
|
async def test_published_is_searchable_draft_is_not(client: AsyncClient) -> None:
|
|
owner = await login(client)
|
|
ws = await create_workspace(client, owner, "检索测试")
|
|
|
|
published = await _knowledge(
|
|
client, owner, ws["id"], "报销发票怎么开", "电子发票原件必须一起寄，复印件不算。"
|
|
)
|
|
parked = await _knowledge(
|
|
client, owner, ws["id"], "供应商账号变更", "变更收款账号需要法人签字并盖章。"
|
|
)
|
|
await _publish(client, owner, ws["id"], published)
|
|
|
|
hits = await _search(client, owner, ws["id"], "发票")
|
|
ids = {hit["item_id"] for hit in hits}
|
|
assert published["id"] in ids
|
|
assert parked["id"] not in ids, "草稿不该出现在检索结果里"
|
|
assert hits[0]["score"] > 0
|
|
assert hits[0]["title"], "检索结果必须带来源标题"
|
|
|
|
# 还没验证过的知识，关键词再怎么对也不该冒出来
|
|
draft_hits = await _search(client, owner, ws["id"], "法人签字")
|
|
assert all(hit["item_id"] != parked["id"] for hit in draft_hits)
|
|
|
|
|
|
async def test_publish_builds_chunks(client: AsyncClient) -> None:
|
|
owner = await login(client)
|
|
ws = await create_workspace(client, owner, "切片测试")
|
|
|
|
body = "\n\n".join(
|
|
[
|
|
"## 背景",
|
|
"十月的报销单据被财务退回两次，第一次是发票复印件，第二次是缺少行程单。" * 20,
|
|
"## 结论",
|
|
"报销必须同时提供电子发票原件与行程单，缺一不可。" * 20,
|
|
]
|
|
)
|
|
doc = await _knowledge(client, owner, ws["id"], "十月报销被退单复盘", body, "postmortem")
|
|
published = await _publish(client, owner, ws["id"], doc)
|
|
assert published["knowledge"]["chunk_count"] >= 2, "长正文应该被切成多片"
|
|
|
|
detail = unwrap(
|
|
await client.get(
|
|
f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}", headers=owner["headers"]
|
|
)
|
|
)
|
|
assert detail["knowledge"]["chunk_count"] >= 2
|
|
|
|
|
|
async def test_reindex_picks_up_edits(client: AsyncClient) -> None:
|
|
owner = await login(client)
|
|
ws = await create_workspace(client, owner, "重建索引")
|
|
|
|
doc = await _knowledge(client, owner, ws["id"], "值班安排", "周一由张三负责。")
|
|
published = await _publish(client, owner, ws["id"], doc)
|
|
assert published["knowledge"]["chunk_count"] == 1
|
|
|
|
# 已发布的内容不许直接改（7003）：产品上的路径是「归档 → 恢复成草稿 → 改」
|
|
archived = unwrap(
|
|
await client.post(
|
|
f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/status",
|
|
json={"action": "archive", "version": published["version"]},
|
|
headers=owner["headers"],
|
|
)
|
|
)
|
|
draft = unwrap(
|
|
await client.post(
|
|
f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/status",
|
|
json={"action": "restore", "version": archived["version"]},
|
|
headers=owner["headers"],
|
|
)
|
|
)
|
|
patched = unwrap(
|
|
await client.patch(
|
|
f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}",
|
|
json={
|
|
"body_md": "周一由张三负责，周三改由李四负责，节假日另行通知。",
|
|
"version": draft["version"],
|
|
},
|
|
headers=owner["headers"],
|
|
)
|
|
)
|
|
# 改完正文但没重建索引，旧切片还在——这就是 reindex 存在的理由
|
|
before = await _search(client, owner, ws["id"], "李四")
|
|
assert all("李四" not in hit["content"] for hit in before)
|
|
|
|
result = unwrap(
|
|
await client.post(
|
|
f"{V1}/workspaces/{ws['id']}/knowledge/{doc['id']}/reindex",
|
|
headers=owner["headers"],
|
|
)
|
|
)
|
|
assert result["chunk_count"] >= 1
|
|
# 仍是草稿，重建了也不该被检索到
|
|
assert result["searchable"] is False
|
|
assert await _search(client, owner, ws["id"], "李四") == []
|
|
|
|
await _publish(client, owner, ws["id"], patched)
|
|
after = await _search(client, owner, ws["id"], "李四")
|
|
assert any("李四" in hit["content"] for hit in after)
|
|
|
|
|
|
async def test_search_is_tenant_scoped(client: AsyncClient) -> None:
|
|
alice = await login(client)
|
|
bob = await login(client)
|
|
ws_a = await create_workspace(client, alice, "甲队")
|
|
ws_b = await create_workspace(client, bob, "乙队")
|
|
|
|
doc = await _knowledge(client, alice, ws_a["id"], "发票规则", "电子发票原件必须一起寄。")
|
|
await _publish(client, alice, ws_a["id"], doc)
|
|
|
|
assert await _search(client, bob, ws_b["id"], "发票") == []
|
|
# 直接拿别人的工作区搜也不行
|
|
denied = await client.get(
|
|
f"{V1}/workspaces/{ws_a['id']}/knowledge/search",
|
|
params={"q": "发票"},
|
|
headers=bob["headers"],
|
|
)
|
|
assert denied.status_code in (403, 404)
|
|
|
|
|
|
async def test_keyword_mode_reports_lexical_score(client: AsyncClient) -> None:
|
|
owner = await login(client)
|
|
ws = await create_workspace(client, owner, "词法检索")
|
|
|
|
doc = await _knowledge(
|
|
client, owner, ws["id"], "顺丰单号核对", "单号 SF1234567890 必须与发货记录一致。"
|
|
)
|
|
await _publish(client, owner, ws["id"], doc)
|
|
|
|
hits = await _search(client, owner, ws["id"], "SF1234567890", mode="keyword")
|
|
assert hits, "带专有编号的查询在词法模式下应该命中"
|
|
assert hits[0]["lexical_score"] > 0
|
|
# keyword 模式里向量权重是 0：总分只由词法分决定
|
|
assert hits[0]["score"] == hits[0]["lexical_score"]
|