公众号的文章也能抓:走 mp.weixin.qq.com 的 DOM(#js_content),
按 data-src 取图,不靠通用抽取器——它会把公众号的图全丢掉。
正文里的外链图下载进我们自己的存储(最多 12 张、单张 8 MB),
地址改成 /attachments/{id}/raw?签名:公众号那边一换图,外链就是一排红叉。
抓取入口谁都能触发,所以只允许公网地址(内网段、回环、链路本地、
保留段、解析不出的域名一律拒绝),否则就是个 SSRF 跳板。
抓不到要说人话:"这个链接没抓下来(原因)。想直接记成任务的话,
加「任务:」前缀再发一次。" 反爬验证页、SPA 空壳都走这条路。
- 新增 services/article.py:extract_url / parse / fetch / store_images / compose_body
- wechat_intake:整条消息是链接时走抓取,落知识草稿,回「已存成知识草稿:标题」
- 依赖 trafilatura==2.0.0(装法见 requirements.txt 注释:必须逐个装再 --no-deps)
- 测试 9 例:链接判定、公众号解析、验证页、通用页裁导航、SSRF 拒绝、图片本地化、失败提示
This commit is contained in:
@@ -17,6 +17,7 @@ from ..models import Member, Workspace
|
||||
from ..models_integrations import BotChannel
|
||||
from ..models_knowledge import KnowledgeDoc
|
||||
from ..models_task import Item
|
||||
from . import article as article_service
|
||||
|
||||
MAX_TITLE = 180
|
||||
MAX_BODY = 20000
|
||||
@@ -30,6 +31,7 @@ HELP_TEXT = (
|
||||
"· 直接说 → 记成任务\n"
|
||||
"· 知识:内容 → 记成知识草稿(也可以写「笔记:」)\n"
|
||||
"· 任务:内容 → 明确记成任务\n"
|
||||
"· 一条链接 → 抓正文存成知识(公众号也行)\n"
|
||||
"首行当标题,后面几行当正文;标题里带 #标签 会自动加上。"
|
||||
)
|
||||
|
||||
@@ -177,6 +179,53 @@ def reply_text(kind: str, item: Item) -> str:
|
||||
return f"已保存任务:{item.title}"
|
||||
|
||||
|
||||
async def _collect_article(
|
||||
session: AsyncSession,
|
||||
channel: BotChannel,
|
||||
url: str,
|
||||
) -> dict[str, Any]:
|
||||
"""一条链接 → 一篇知识草稿(正文 + 转存后的图)。
|
||||
|
||||
抓不到就如实说,并把原链接留着:用户多半还会想手动点开看看,
|
||||
回一句"失败了"再让他重发一遍是最糟的体验。
|
||||
"""
|
||||
creator = await creator_id(session, channel)
|
||||
try:
|
||||
fetched = await article_service.fetch_article(url)
|
||||
except article_service.ArticleError as exc:
|
||||
return {
|
||||
"action": "ignored",
|
||||
"item_id": None,
|
||||
"reply": f"这个链接没抓下来({exc})。想直接记成任务的话,加「任务:」前缀再发一次。",
|
||||
}
|
||||
|
||||
item = await create_knowledge(
|
||||
session,
|
||||
ws_id=channel.workspace_id,
|
||||
creator=creator,
|
||||
title=fetched.title,
|
||||
body=article_service.compose_body(fetched),
|
||||
tags=[],
|
||||
)
|
||||
body_md, saved = await article_service.store_images(
|
||||
session,
|
||||
ws_id=channel.workspace_id,
|
||||
item_id=item.id,
|
||||
uploader_id=creator,
|
||||
body_md=item.body_md,
|
||||
)
|
||||
item.body_md = body_md
|
||||
await session.flush()
|
||||
|
||||
# 不写"已收藏",写清楚抓到了什么:有没有正文、有没有图,一眼能判断值不值得去补。
|
||||
tail = f",连图 {len(saved)} 张" if saved else ""
|
||||
return {
|
||||
"action": "knowledge",
|
||||
"item_id": item.id,
|
||||
"reply": f"已存成知识草稿:{item.title}{tail}",
|
||||
}
|
||||
|
||||
|
||||
async def handle_text(
|
||||
session: AsyncSession,
|
||||
channel: BotChannel,
|
||||
@@ -188,6 +237,9 @@ async def handle_text(
|
||||
kind, body = split_command(text.strip())
|
||||
if kind == "help":
|
||||
return {"action": "help", "item_id": None, "reply": HELP_TEXT}
|
||||
link = article_service.extract_url(body)
|
||||
if link:
|
||||
return await _collect_article(session, channel, link)
|
||||
body, tags = split_tags(body)
|
||||
title, rest = split_title(body)
|
||||
if not title:
|
||||
|
||||
Reference in New Issue
Block a user