Steven
690779c5fb
CI / backend (push) Canceled after 0s
用户发来 https://mp.weixin.qq.com/s/ZEOsi4R4JcmZ_mhS6qpcww，存进库的正文只有 67 个字符 （"原文链接 + 一个句号"）。复盘发现公众号换了渲染方式：新版页面把整篇塞进 window.cgiDataNew 这个 JS 对象由前端渲染，页面里根本没有 id="js_content"； 三种 UA 拿到的字节数完全一样，也不是 UA 的问题。 - 从 JS 字符串字面量里读正文（content_noencode / desc），按 \xNN、\uNNNN 反转义， 话题标签是双重转义过的 HTML，先 unescape 再走 DOM。 - 配图字段排在文字之后，所以前后各找 2500 字符合图；剩下没用过的 mmbiz 图按 12 张上限补在末尾（图文型文章靠就近配对会漏）。 - 解析顺序：旧 DOM → 新版 JS 数据 → 通用抽取，一层层退。 实测两篇：2373 字符 + 12 张图、2772 字符 + 13 张图，作者也都取到了。 同时补上用户提的两件： - 知识能删了：DELETE /knowledge/{id}?confirm=true，软删 + 真删检索切片， 只有作者或 owner/admin 能删；列表卡片（悬停出现）和阅读页动作行各一个入口。 删完详情也回 7001，否则收藏的链接还能打开一篇已删除的知识。 - 分类下面有标签清单了：GET /knowledge/tags 按标签数"有多少条知识用了它"， 侧栏渲染成 #标签 + 条数，点一下筛选、再点取消。
285 lines
11 KiB
Python
285 lines
11 KiB
Python
"""微信里发链接 → 抓成知识。
|
|
|
|
不联网：`fetch_article` 与 `store_images` 用 monkeypatch 顶掉，验的是解析规则与落库；
|
|
真抓取（trafilatura / 公众号 DOM）在 `parse_article` 的用例里用固定 HTML 覆盖。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from httpx import AsyncClient
|
|
from sqlalchemy import select
|
|
|
|
from app.db import SessionFactory, bind_tenant
|
|
from app.models_integrations import BotChannel
|
|
from app.models_knowledge import KnowledgeDoc
|
|
from app.models_task import Item
|
|
from app.services import article as article_service
|
|
from app.services.wechat_intake import handle_text
|
|
|
|
from .conftest import create_workspace, login
|
|
|
|
WECHAT_HTML = """
|
|
<html><head>
|
|
<meta property="og:title" content="公众号标题兜底">
|
|
</head><body>
|
|
<h1 id="activity-name"> 回滚前先停灰度：一次线上事故的复盘 </h1>
|
|
<a id="js_name">达道技术</a>
|
|
<div id="js_content">
|
|
<section><p>事故发生在周五晚高峰，灰度流量上涨后错误率飙升。</p></section>
|
|
<section><p>第一步先停灰度，把流量打回稳定版本。</p>
|
|
<p><img data-src="https://mmbiz.qpic.cn/mmbiz_png/abc/640?wx_fmt=png" /></p>
|
|
</section>
|
|
<section><section><p>第二步核对回滚后的核心指标。</p></section></section>
|
|
</div>
|
|
<div id="js_pc_qr_code">扫码关注</div>
|
|
</body></html>
|
|
"""
|
|
|
|
BLOCKED_HTML = """
|
|
<html><body><div class="weui-msg"><p>环境异常</p><p>完成验证后即可继续访问</p></div></body></html>
|
|
"""
|
|
|
|
# 新版公众号页面：没有 #js_content，正文全在 window.cgiDataNew 里；
|
|
# 换行是 \x0a，话题标签是双重转义过的 HTML 片段，图片字段排在文字前面。
|
|
# 正文抽成常量：desc 与 content_noencode 同文（真实页面也是这样，后者更权威），
|
|
# 写成一行会超行宽，所以拆开拼。
|
|
_WECHAT_DATA_BODY = (
|
|
r"食材： 蒜瓣500g、小米辣60g\x0a\x0a步骤：\x0a1\u20e3蒜瓣打碎，"
|
|
r"放在冷水里泡10分钟（泡一下蒜不会发苦）\x0a2\u20e3油烧热，转小火把蒜泥炒香"
|
|
r"\x0a\x0a注意\x26lt;a class=\x26quot;wx_topic_link\x26quot;\x26gt;"
|
|
r"#调料\x26lt;/a\x26gt;"
|
|
)
|
|
WECHAT_DATA_HTML = (
|
|
"<html><head>\n"
|
|
'<meta property="og:title" content="自制万能蒜蓉酱" />\n'
|
|
"</head><body>\n<script>\n window.cgiDataNew = {\n"
|
|
" nick_name: '破冰记的随笔',\n item_list: [{\n"
|
|
" cdn_url: 'http://mmbiz.qpic.cn/mmbiz_jpg/xyz/640?wx_fmt=jpeg',\n"
|
|
" title: '自制万能蒜蓉酱',\n"
|
|
" desc: '" + _WECHAT_DATA_BODY + "',\n"
|
|
" content_noencode: '" + _WECHAT_DATA_BODY + "'\n"
|
|
" }]\n };\n</script>\n</body></html>\n"
|
|
)
|
|
|
|
|
|
def test_extract_url_only_when_message_is_a_link() -> None:
|
|
assert article_service.extract_url("https://mp.weixin.qq.com/s/abc") == (
|
|
"https://mp.weixin.qq.com/s/abc"
|
|
)
|
|
# 前后有点标点也算，手机上复制链接常带标点
|
|
assert article_service.extract_url(" https://a.com/x 。") == "https://a.com/x"
|
|
# 带了别的内容就不是"发链接"，它是想记条任务
|
|
assert article_service.extract_url("看下 https://a.com/x 这个方案") is None
|
|
assert article_service.extract_url("明天下午三点开会") is None
|
|
|
|
|
|
def test_wechat_article_parses_title_body_and_images() -> None:
|
|
got = article_service.parse_article(WECHAT_HTML, "https://mp.weixin.qq.com/s/abc")
|
|
|
|
assert got.title == "回滚前先停灰度：一次线上事故的复盘"
|
|
assert got.author == "达道技术"
|
|
assert got.site == "微信公众号"
|
|
assert "灰度流量上涨后错误率飙升" in got.body_md
|
|
# 图片取 data-src，不能丢
|
|
assert "https://mmbiz.qpic.cn/mmbiz_png/abc/640?wx_fmt=png" in got.body_md
|
|
# 正文不能重复输出（嵌套 section 是最容易重复的地方）
|
|
assert got.body_md.count("第一步先停灰度") == 1
|
|
assert got.body_md.count("第二步核对回滚后的核心指标") == 1
|
|
# 页面底部二维码之类的噪声不该进来
|
|
assert "扫码关注" not in got.body_md
|
|
|
|
|
|
def test_new_wechat_page_reads_js_data() -> None:
|
|
"""新版页面没有 #js_content：正文在 cgiDataNew 的 JS 字符串里，得能读出来。"""
|
|
got = article_service.parse_article(WECHAT_DATA_HTML, "https://mp.weixin.qq.com/s/abc")
|
|
|
|
assert got.title == "自制万能蒜蓉酱"
|
|
assert got.author == "破冰记的随笔"
|
|
assert "蒜瓣打碎" in got.body_md
|
|
# \x0a 要还原成换行，否则整篇挤成一行
|
|
assert "\n" in got.body_md
|
|
# 双重转义的话题标签还原成文字，不能留下 <a 这种碎片
|
|
assert "#调料" in got.body_md
|
|
assert "<" not in got.body_md
|
|
# 配图要跟着正文进来（正文里换成本地地址之前，先能取到这张图）
|
|
assert "mmbiz.qpic.cn/mmbiz_jpg/xyz/640" in got.body_md
|
|
# desc 与 content_noencode 是同一段文字，不能重复两遍
|
|
assert got.body_md.count("蒜瓣打碎") == 1
|
|
|
|
|
|
def test_blocked_wechat_page_is_reported() -> None:
|
|
with pytest.raises(article_service.ArticleError):
|
|
article_service.parse_article(BLOCKED_HTML, "https://mp.weixin.qq.com/s/abc")
|
|
|
|
|
|
def test_generic_page_gets_chrome_trimmed() -> None:
|
|
html = """<html><body>
|
|
<nav><a>首页</a><a>产品</a><a>关于我们</a></nav>
|
|
<article>
|
|
<h1>怎么把知识管起来</h1>
|
|
<p>先把重复回答的问题记下来，这一步最便宜，也最容易坚持。</p>
|
|
<p>第二步才是建目录，目录只服务于找得到，不要为了好看去分二十个类。</p>
|
|
</article>
|
|
<footer><a>Sign In</a><a>Privacy</a></footer>
|
|
</body></html>"""
|
|
got = article_service.parse_article(html, "https://example.com/post")
|
|
assert "先把重复回答的问题记下来" in got.body_md
|
|
assert "Sign In" not in got.body_md
|
|
|
|
|
|
def test_compose_body_keeps_source_line() -> None:
|
|
got = article_service.Article(
|
|
url="https://mp.weixin.qq.com/s/abc",
|
|
title="标题",
|
|
body_md="正文",
|
|
site="微信公众号",
|
|
author="达道技术",
|
|
)
|
|
body = article_service.compose_body(got)
|
|
assert body.startswith("> 原文：<https://mp.weixin.qq.com/s/abc>")
|
|
assert "微信公众号 · 达道技术" in body
|
|
assert body.endswith("正文")
|
|
|
|
|
|
async def test_private_address_is_refused() -> None:
|
|
"""这个入口谁都能触发，不能当 SSRF 跳板。"""
|
|
for url in ("http://127.0.0.1:8200/version", "http://192.168.8.188:8200/api"):
|
|
with pytest.raises(article_service.ArticleError):
|
|
await article_service.fetch_article(url)
|
|
|
|
|
|
async def test_link_message_becomes_knowledge(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
user = await login(client)
|
|
ws = await create_workspace(client, user, "收链接")
|
|
|
|
async def fake_fetch(url: str) -> article_service.Article:
|
|
return article_service.Article(
|
|
url=url,
|
|
title="回滚前先停灰度",
|
|
body_md="先停流量，再退镜像版本。",
|
|
site="微信公众号",
|
|
author="达道技术",
|
|
)
|
|
|
|
async def fake_store(session: Any, **kwargs: Any) -> tuple[str, list[Any]]:
|
|
return kwargs["body_md"], []
|
|
|
|
monkeypatch.setattr(article_service, "fetch_article", fake_fetch)
|
|
monkeypatch.setattr(article_service, "store_images", fake_store)
|
|
|
|
ws_id = uuid.UUID(ws["id"])
|
|
async with SessionFactory() as session:
|
|
await bind_tenant(session, ws_id, uuid.UUID(user["user_id"]))
|
|
channel = BotChannel(
|
|
workspace_id=ws_id,
|
|
kind="wechat_clawbot",
|
|
name="微信",
|
|
status="active",
|
|
created_by=uuid.UUID(user["user_id"]),
|
|
)
|
|
session.add(channel)
|
|
await session.flush()
|
|
|
|
result = await handle_text(
|
|
session,
|
|
channel,
|
|
"https://mp.weixin.qq.com/s/abc",
|
|
base_url="http://site",
|
|
)
|
|
await session.commit()
|
|
|
|
assert result["action"] == "knowledge"
|
|
assert result["reply"] == "已存成知识草稿：回滚前先停灰度"
|
|
async with SessionFactory() as session:
|
|
await bind_tenant(session, ws_id, uuid.UUID(user["user_id"]))
|
|
item = await session.get(Item, result["item_id"])
|
|
assert item is not None
|
|
assert item.kind == "knowledge"
|
|
assert item.status == "draft"
|
|
assert "> 原文：<https://mp.weixin.qq.com/s/abc>" in (item.body_md or "")
|
|
doc = (
|
|
await session.execute(select(KnowledgeDoc).where(KnowledgeDoc.item_id == item.id))
|
|
).scalar_one()
|
|
assert doc.status_hint == "draft"
|
|
|
|
|
|
async def test_link_failure_is_reported_not_silent(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
user = await login(client)
|
|
ws = await create_workspace(client, user, "收链接")
|
|
|
|
async def fake_fetch(url: str) -> article_service.Article: # noqa: ARG001
|
|
raise article_service.ArticleError("对方站点返回 403")
|
|
|
|
monkeypatch.setattr(article_service, "fetch_article", fake_fetch)
|
|
|
|
ws_id = uuid.UUID(ws["id"])
|
|
async with SessionFactory() as session:
|
|
await bind_tenant(session, ws_id, uuid.UUID(user["user_id"]))
|
|
channel = BotChannel(
|
|
workspace_id=ws_id,
|
|
kind="wechat_clawbot",
|
|
name="微信",
|
|
status="active",
|
|
created_by=uuid.UUID(user["user_id"]),
|
|
)
|
|
session.add(channel)
|
|
await session.flush()
|
|
|
|
result = await handle_text(
|
|
session, channel, "https://example.com/a", base_url="http://site"
|
|
)
|
|
await session.commit()
|
|
|
|
assert result["action"] == "ignored"
|
|
assert "没抓下来" in result["reply"]
|
|
assert "任务：" in result["reply"]
|
|
|
|
|
|
async def test_images_are_copied_into_our_storage(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""文章里的图要落到我们自己的存储里：公众号一改图，正文就是一排红叉。"""
|
|
user = await login(client)
|
|
ws = await create_workspace(client, user, "存图")
|
|
ws_id = uuid.UUID(ws["id"])
|
|
|
|
async def fake_download(client: Any, url: str) -> tuple[bytes, str]: # noqa: ARG001
|
|
return b"\x89PNG\r\n\x1a\nfake", "image/png"
|
|
|
|
monkeypatch.setattr(article_service, "_download_image", fake_download)
|
|
|
|
body = "看图\n\n"
|
|
async with SessionFactory() as session:
|
|
await bind_tenant(session, ws_id, uuid.UUID(user["user_id"]))
|
|
item = Item(
|
|
workspace_id=ws_id,
|
|
kind="knowledge",
|
|
title="带图的一篇",
|
|
body_md=body,
|
|
status="draft",
|
|
creator_id=uuid.UUID(user["user_id"]),
|
|
)
|
|
session.add(item)
|
|
await session.flush()
|
|
rewritten, rows = await article_service.store_images(
|
|
session,
|
|
ws_id=ws_id,
|
|
item_id=item.id,
|
|
uploader_id=uuid.UUID(user["user_id"]),
|
|
body_md=body,
|
|
)
|
|
await session.commit()
|
|
|
|
assert len(rows) == 1
|
|
assert rows[0].filename.startswith("图1.")
|
|
assert "attachment" in rewritten and "/raw?" in rewritten
|
|
assert "mmbiz.qpic.cn" not in rewritten
|