feat(knowledge): 语义检索打通——切片、混合打分、只搜已发布
CI / backend (push) Canceled after 0s

之前知识只能"存",不能"找"。这一版把链路补上:切片 → 向量 → 打分 → 带来源返回。

- chunk 表 + 迁移 0006:正文按标题归段切、代码块整块保留、超长按句切、
  相邻片重叠 80 字;RLS 与 GRANT 与其它租户表一致
- 向量提供方做成协议:内置 hash(字符二元组,零依赖可复现,作为基线)
  与 OpenAI 兼容端点两种,换实现不动业务代码
- 检索是混合打分:向量 0.6 + pg_trgm 0.4。专有编号、单号这类必须原样命中的词
  靠词法兜,换个说法表达的同一件事靠向量兜
- 三条纪律:只搜 published(过滤写在 SQL 里)、低于阈值宁可不返回、
  每条命中都带来源标题与 item_id
- 发布时自动建索引,改完正文用 POST /knowledge/{id}/reindex 重建
- Web 知识页加检索框与命中卡片;新增 e2e/retrieval.mjs 6 项

为什么不是 pgvector:本机 Docker 起不来、fnsvr-01 没有该扩展且明确不动。
先用 real[] + SQL 余弦把链路跑通,将来换列类型与索引即可,接口不变。

后端 79 passed,e2e smoke/knowledge/richtext/retrieval 全绿。
This commit is contained in:
dadao
2026-10-29 04:28:09 +08:00
parent d509f0c335
commit d079564d6c
16 changed files with 1126 additions and 22 deletions
+120
View File
@@ -0,0 +1,120 @@
/**
* 检索 e2e:发布的知识能被搜到,草稿搜不到,结果能点回原文。
*
* 只验证"用户能不能用上",切片与打分逻辑由后端测试覆盖。
*/
import { chromium } from "playwright-core";
const WEB = process.env.WEB ?? "http://127.0.0.1:5180";
const API = process.env.API ?? "http://127.0.0.1:8100/api/v1";
const stamp = String(Date.now()).slice(-7);
const phone = `134${stamp}0`;
let failures = 0;
function check(name, ok, extra = "") {
console.log(`${ok ? "PASS" : "FAIL"} ${name}${extra ? ` — ${extra}` : ""}`);
if (!ok) failures += 1;
}
async function call(method, path, body, token) {
const res = await fetch(`${API}${path}`, {
method,
headers: {
"content-type": "application/json",
...(token ? { authorization: `Bearer ${token}` } : {}),
...(method === "GET" ? {} : { "idempotency-key": `e2e-${Math.random()}` }),
},
body: body ? JSON.stringify(body) : undefined,
});
const text = await res.text();
const json = text ? JSON.parse(text) : {};
if (json.code !== 0) throw new Error(`${method} ${path} -> ${json.code} ${json.message}`);
return json.data;
}
const browser = await chromium.launch({
executablePath:
process.env.CHROME ??
`${process.env.LOCALAPPDATA}\\ms-playwright\\chromium-1223\\chrome-win64\\chrome.exe`,
});
const consoleErrors = [];
const context = await browser.newContext({ viewport: { width: 1280, height: 900 } });
const page = await context.newPage();
page.on("console", (m) => {
if (m.type() === "error") consoleErrors.push(m.text());
});
page.on("response", (r) => {
if (r.status() >= 400) consoleErrors.push(`${r.status()} ${r.url()}`);
});
try {
await call("POST", "/auth/sms/send", { phone });
const login = await call("POST", "/auth/login", { phone, code: "123456" });
const token = login.tokens.access_token;
const ws = await call("POST", "/workspaces", { name: `检索验收 ${stamp}` }, token);
const published = await call(
"POST",
`/workspaces/${ws.id}/knowledge`,
{
title: "快递单号对不上怎么处理",
doc_type: "howto",
body_md:
"## 步骤\n\n先在顺丰后台按手机号反查单号,再和发货记录逐一核对。\n\n如果差超过两天,直接找客服开异常件工单。",
},
token,
);
await call(
"POST",
`/workspaces/${ws.id}/knowledge/${published.id}/status`,
{ action: "publish", version: published.version },
token,
);
const draft = await call(
"POST",
`/workspaces/${ws.id}/knowledge`,
{ title: "还没核对的草稿", doc_type: "note", body_md: "这条写的是异常件的赔付口径,尚未定稿。" },
token,
);
await page.addInitScript(
([access, refresh]) => {
localStorage.setItem("dadao.access", access);
localStorage.setItem("dadao.refresh", refresh);
},
[token, login.tokens.refresh_token],
);
await page.goto(`${WEB}/w/${ws.id}/knowledge`, { waitUntil: "networkidle" });
const box = page.getByPlaceholder(/搜标题或正文/);
await box.fill("异常件单号核对");
await box.press("Enter");
await page.getByText(/命中 \d+ 段/).waitFor({ timeout: 10_000 });
const summary = await page.getByText(/命中 \d+ 段/).first().textContent();
check("检索返回了结果", /命中 [1-9]\d* 段/.test(summary ?? ""), summary ?? "");
check("命中里带标题", await page.getByText("快递单号对不上怎么处理", { exact: false }).first().isVisible());
check("草稿不会出现在检索结果里", !(await page.getByText("还没核对的草稿", { exact: false }).count()));
await page.screenshot({ path: "e2e/artifacts/retrieval-hits.png", fullPage: false });
await page.getByRole("button", { name: "打开原文", exact: true }).first().click();
await page.getByText(/已发布 · 版本 \d+/, { exact: false }).waitFor({ timeout: 10_000 });
check("命中能点回原文", true);
await page.getByRole("button", { name: "关闭", exact: true }).click();
await box.fill("量子纠缠");
await box.press("Enter");
await page.getByText("没有检索到相关内容", { exact: false }).waitFor({ timeout: 10_000 });
check("不相干的关键词宁可返回空", true);
check("浏览器控制台与网络无报错", consoleErrors.length === 0, consoleErrors.slice(0, 3).join(" | "));
} catch (err) {
failures += 1;
console.log(`FAIL 脚本异常 — ${err.message}`);
await page.screenshot({ path: "e2e/artifacts/retrieval-failure.png" }).catch(() => {});
console.log((await page.textContent("body"))?.slice(0, 600));
} finally {
await browser.close();
console.log(failures === 0 ? "\nALL PASS" : `\n${failures} FAILED`);
process.exit(failures === 0 ? 0 : 1);
}
+13
View File
@@ -157,3 +157,16 @@ export interface Knowledge extends Item {
attachment_count?: number;
activities?: Activity[];
}
/** 一条检索命中:切片级结果,带来源,点得回原文。 */
export interface SearchHit {
item_id: string;
title: string;
doc_type: string;
idx: number;
heading: string | null;
content: string;
score: number;
vector_score: number;
lexical_score: number;
}
+87 -4
View File
@@ -14,7 +14,7 @@ import {
StatusPill,
} from "../components/ui";
import { ApiError, api } from "../lib/client";
import type { Attachment, Knowledge, KnowledgeStatus } from "../lib/types";
import type { Attachment, Knowledge, KnowledgeStatus, SearchHit } from "../lib/types";
const TABS = [
{ key: "draft", label: "待核对", statuses: ["draft", "in_review"] },
@@ -63,6 +63,8 @@ export function KnowledgePage() {
const [openId, setOpenId] = useState<string | null>(null);
const [adding, setAdding] = useState(false);
const [draftTitle, setDraftTitle] = useState("");
const [hits, setHits] = useState<SearchHit[] | null>(null);
const [searching, setSearching] = useState(false);
const reqId = useRef(0);
const load = useCallback(
@@ -99,6 +101,32 @@ export function KnowledgePage() {
void load(null);
}, [load]);
/**
* 语义检索走的是切片接口,和上面的列表筛选不是一回事:
* 列表按标题/正文做子串匹配,检索按向量 + trigram 打分,只覆盖已发布的知识。
* 所以不把 q 直接接到列表上——两套结果混在一起会让人搞不清命中在哪。
*/
async function runSearch() {
const keyword = q.trim();
if (!keyword) {
setHits(null);
return;
}
setSearching(true);
setError(null);
try {
const query = new URLSearchParams({ q: keyword, limit: "12" });
const page = await api.get<{ items: SearchHit[] }>(
`/workspaces/${wsId}/knowledge/search?${query.toString()}`,
);
setHits(page.items);
} catch (err) {
setError(err instanceof ApiError ? err.message : "检索失败");
} finally {
setSearching(false);
}
}
async function addNote() {
const title = draftTitle.trim();
if (!title) return;
@@ -133,11 +161,21 @@ export function KnowledgePage() {
</span>
<input
className="input h-9 w-48 pl-8 text-sm"
placeholder="搜标题或正文…"
placeholder="搜标题或正文…(回车检索)"
value={q}
onChange={(e) => setQ(e.target.value)}
onChange={(e) => {
setQ(e.target.value);
if (!e.target.value.trim()) setHits(null);
}}
onKeyDown={(e) => {
if (e.key === "Enter") void runSearch();
}}
/>
</div>
<Button variant="ghost" disabled={searching || !q.trim()} onClick={() => void runSearch()}>
<Search size={14} />
{searching ? "检索中…" : "检索"}
</Button>
<Button variant="primary" onClick={() => setAdding((v) => !v)}>
<Plus size={14} />
记一条
@@ -204,7 +242,52 @@ export function KnowledgePage() {
</div>
)}
{loading && docs.length === 0 ? (
{hits !== null ? (
<section className="space-y-2">
<div className="flex items-baseline justify-between px-1">
<p className="text-xs text-ink-500">
在已发布的知识里检索「{q.trim()}」,命中 {hits.length} 段
</p>
<button className="link text-2xs" onClick={() => setHits(null)}>
退出检索
</button>
</div>
{hits.length === 0 ? (
<div className="card">
<EmptyState
icon={<Search size={18} />}
title="没有检索到相关内容"
hint="只检索已发布的知识。换成更具体的说法,或者先把手头的草稿核对发布。"
/>
</div>
) : (
hits.map((hit) => (
<article key={`${hit.item_id}-${hit.idx}`} className="card p-4">
<div className="flex flex-wrap items-center gap-1.5">
<span className="chip bg-primary-50 text-primary-700">
{DOC_TYPE_LABEL[hit.doc_type] ?? hit.doc_type}
</span>
{hit.heading ? (
<span className="chip bg-paper-200 text-ink-600">{hit.heading}</span>
) : null}
<span className="ml-auto text-2xs text-ink-400">
相关度 {hit.score.toFixed(2)}
</span>
</div>
<p className="mt-2 text-sm font-medium text-ink-900">{hit.title}</p>
<p className="mt-1.5 line-clamp-3 whitespace-pre-wrap text-xs leading-relaxed text-ink-600">
{hit.content}
</p>
<div className="mt-2 text-right">
<Button variant="ghost" size="sm" onClick={() => setOpenId(hit.item_id)}>
打开原文
</Button>
</div>
</article>
))
)}
</section>
) : loading && docs.length === 0 ? (
<div className="card overflow-hidden">
<SkeletonRows rows={3} />
</div>