CI / backend (push) Canceled after 0s
公众号的文章也能抓:走 mp.weixin.qq.com 的 DOM(#js_content),
按 data-src 取图,不靠通用抽取器——它会把公众号的图全丢掉。
正文里的外链图下载进我们自己的存储(最多 12 张、单张 8 MB),
地址改成 /attachments/{id}/raw?签名:公众号那边一换图,外链就是一排红叉。
抓取入口谁都能触发,所以只允许公网地址(内网段、回环、链路本地、
保留段、解析不出的域名一律拒绝),否则就是个 SSRF 跳板。
抓不到要说人话:"这个链接没抓下来(原因)。想直接记成任务的话,
加「任务:」前缀再发一次。" 反爬验证页、SPA 空壳都走这条路。
- 新增 services/article.py:extract_url / parse / fetch / store_images / compose_body
- wechat_intake:整条消息是链接时走抓取,落知识草稿,回「已存成知识草稿:标题」
- 依赖 trafilatura==2.0.0(装法见 requirements.txt 注释:必须逐个装再 --no-deps)
- 测试 9 例:链接判定、公众号解析、验证页、通用页裁导航、SSRF 拒绝、图片本地化、失败提示
275 lines
8.6 KiB
Python
275 lines
8.6 KiB
Python
"""把一句话变成任务或知识草稿。
|
||
|
||
微信只负责把文本送进来,怎么落成事项是这里的事,接口和后台轮询共用同一份逻辑,
|
||
免得"网页里发"和"机器人里发"落成两种东西。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import uuid
|
||
from typing import Any
|
||
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from ..errors import AppError, ErrorCode
|
||
from ..models import Member, Workspace
|
||
from ..models_integrations import BotChannel
|
||
from ..models_knowledge import KnowledgeDoc
|
||
from ..models_task import Item
|
||
from . import article as article_service
|
||
|
||
MAX_TITLE = 180
|
||
MAX_BODY = 20000
|
||
|
||
KNOWLEDGE_PREFIXES = ("知识", "笔记", "note", "knowledge", "kb")
|
||
TASK_PREFIXES = ("任务", "待办", "todo", "task")
|
||
HELP_PREFIXES = ("?", "?", "帮助", "help", "用法")
|
||
|
||
HELP_TEXT = (
|
||
"发什么都能用,写法:\n"
|
||
"· 直接说 → 记成任务\n"
|
||
"· 知识:内容 → 记成知识草稿(也可以写「笔记:」)\n"
|
||
"· 任务:内容 → 明确记成任务\n"
|
||
"· 一条链接 → 抓正文存成知识(公众号也行)\n"
|
||
"首行当标题,后面几行当正文;标题里带 #标签 会自动加上。"
|
||
)
|
||
|
||
SEPARATORS = (":", ":", " ", "\u3000")
|
||
|
||
|
||
def split_command(text: str) -> tuple[str, str]:
|
||
"""返回 (kind, 去掉前缀后的正文)。"""
|
||
head, _, rest = text.partition("\n")
|
||
probe = head.strip().lower()
|
||
for prefix in HELP_PREFIXES:
|
||
if probe == prefix or probe.startswith(f"{prefix}:") or probe.startswith(f"{prefix}:"):
|
||
return "help", rest
|
||
for prefix in KNOWLEDGE_PREFIXES:
|
||
for sep in SEPARATORS:
|
||
if probe.startswith(f"{prefix}{sep}"):
|
||
offset = len(prefix) + len(sep)
|
||
return "knowledge", (head.strip()[offset:] + ("\n" + rest if rest else "")).strip()
|
||
for prefix in TASK_PREFIXES:
|
||
for sep in SEPARATORS:
|
||
if probe.startswith(f"{prefix}{sep}"):
|
||
offset = len(prefix) + len(sep)
|
||
return "task", (head.strip()[offset:] + ("\n" + rest if rest else "")).strip()
|
||
return "task", text
|
||
|
||
|
||
def split_tags(body: str) -> tuple[str, list[str]]:
|
||
"""把 `#标签` 摘出来。只认行首和空白后的 #,免得把 "C#" 吃掉。"""
|
||
tags: list[str] = []
|
||
kept: list[str] = []
|
||
for line in body.splitlines():
|
||
rest: list[str] = []
|
||
for word in line.split(" "):
|
||
if word.startswith("#") and len(word) > 1 and not word[1:].isdigit():
|
||
name = word[1:].strip(",,。.;;")
|
||
if name and name.lower() not in {t.lower() for t in tags}:
|
||
tags.append(name)
|
||
continue
|
||
rest.append(word)
|
||
kept.append(" ".join(rest).rstrip())
|
||
return "\n".join(kept).strip(), tags
|
||
|
||
|
||
def split_title(body: str) -> tuple[str, str]:
|
||
lines = body.splitlines()
|
||
if not lines:
|
||
return "", ""
|
||
return lines[0].strip()[:MAX_TITLE], "\n".join(lines[1:]).strip()[:MAX_BODY]
|
||
|
||
|
||
async def creator_id(session: AsyncSession, channel: BotChannel) -> uuid.UUID:
|
||
"""消息记在谁名下:通道创建者;人没了就退回工作区 owner。"""
|
||
if channel.created_by is not None:
|
||
row = await session.execute(
|
||
select(Member.id).where(
|
||
Member.workspace_id == channel.workspace_id,
|
||
Member.user_id == channel.created_by,
|
||
Member.status == "active",
|
||
)
|
||
)
|
||
if row.first() is not None:
|
||
return channel.created_by
|
||
workspace = await session.get(Workspace, channel.workspace_id)
|
||
if workspace is None: # 理论上不可能:FK 是 CASCADE
|
||
raise AppError(ErrorCode.BOT_TOKEN_INVALID, "接入通道不存在")
|
||
return workspace.owner_id
|
||
|
||
|
||
async def create_task(
|
||
session: AsyncSession,
|
||
*,
|
||
ws_id: uuid.UUID,
|
||
creator: uuid.UUID,
|
||
title: str,
|
||
body: str,
|
||
tags: list[str],
|
||
) -> Item:
|
||
item = Item(
|
||
workspace_id=ws_id,
|
||
kind="task",
|
||
title=title,
|
||
body_md=body,
|
||
status="inbox",
|
||
priority=0,
|
||
creator_id=creator,
|
||
)
|
||
session.add(item)
|
||
await session.flush()
|
||
from ..api.items import _log, _set_tags # 延迟导入:services 不该在模块级依赖 api
|
||
|
||
if tags:
|
||
await _set_tags(session, ws_id, item, tags)
|
||
await _log(session, item, creator, "created", {"via": "wechat_clawbot"})
|
||
return item
|
||
|
||
|
||
async def create_knowledge(
|
||
session: AsyncSession,
|
||
*,
|
||
ws_id: uuid.UUID,
|
||
creator: uuid.UUID,
|
||
title: str,
|
||
body: str,
|
||
tags: list[str],
|
||
) -> Item:
|
||
item = Item(
|
||
workspace_id=ws_id,
|
||
kind="knowledge",
|
||
title=title,
|
||
body_md=body,
|
||
status="draft",
|
||
creator_id=creator,
|
||
)
|
||
session.add(item)
|
||
await session.flush()
|
||
session.add(
|
||
KnowledgeDoc(
|
||
item_id=item.id,
|
||
workspace_id=ws_id,
|
||
author_id=creator,
|
||
source_item_id=None,
|
||
source_kind="manual",
|
||
doc_type="note",
|
||
status_hint="draft",
|
||
generated_by="human",
|
||
)
|
||
)
|
||
await session.flush()
|
||
from ..api.items import _log, _set_tags # 延迟导入,理由同上
|
||
|
||
if tags:
|
||
await _set_tags(session, ws_id, item, tags)
|
||
await _log(session, item, creator, "knowledge_created", {"via": "wechat_clawbot"})
|
||
return item
|
||
|
||
|
||
def reply_text(kind: str, item: Item) -> str:
|
||
"""回话只说"收下了",不带链接。
|
||
|
||
链接是站点地址,可"站点地址"对手机不一定打得开(内网 IP、端口、还没上 HTTPS 的域名),
|
||
发过去反而像坏消息。要看详情回网页里看就行。
|
||
"""
|
||
if kind == "knowledge":
|
||
return f"已保存知识草稿:{item.title}"
|
||
return f"已保存任务:{item.title}"
|
||
|
||
|
||
async def _collect_article(
|
||
session: AsyncSession,
|
||
channel: BotChannel,
|
||
url: str,
|
||
) -> dict[str, Any]:
|
||
"""一条链接 → 一篇知识草稿(正文 + 转存后的图)。
|
||
|
||
抓不到就如实说,并把原链接留着:用户多半还会想手动点开看看,
|
||
回一句"失败了"再让他重发一遍是最糟的体验。
|
||
"""
|
||
creator = await creator_id(session, channel)
|
||
try:
|
||
fetched = await article_service.fetch_article(url)
|
||
except article_service.ArticleError as exc:
|
||
return {
|
||
"action": "ignored",
|
||
"item_id": None,
|
||
"reply": f"这个链接没抓下来({exc})。想直接记成任务的话,加「任务:」前缀再发一次。",
|
||
}
|
||
|
||
item = await create_knowledge(
|
||
session,
|
||
ws_id=channel.workspace_id,
|
||
creator=creator,
|
||
title=fetched.title,
|
||
body=article_service.compose_body(fetched),
|
||
tags=[],
|
||
)
|
||
body_md, saved = await article_service.store_images(
|
||
session,
|
||
ws_id=channel.workspace_id,
|
||
item_id=item.id,
|
||
uploader_id=creator,
|
||
body_md=item.body_md,
|
||
)
|
||
item.body_md = body_md
|
||
await session.flush()
|
||
|
||
# 不写"已收藏",写清楚抓到了什么:有没有正文、有没有图,一眼能判断值不值得去补。
|
||
tail = f",连图 {len(saved)} 张" if saved else ""
|
||
return {
|
||
"action": "knowledge",
|
||
"item_id": item.id,
|
||
"reply": f"已存成知识草稿:{item.title}{tail}",
|
||
}
|
||
|
||
|
||
async def handle_text(
|
||
session: AsyncSession,
|
||
channel: BotChannel,
|
||
text: str,
|
||
*,
|
||
base_url: str,
|
||
) -> dict[str, Any]:
|
||
"""一段文本 → (action, item_id, reply)。调用方负责提交事务与留痕。"""
|
||
kind, body = split_command(text.strip())
|
||
if kind == "help":
|
||
return {"action": "help", "item_id": None, "reply": HELP_TEXT}
|
||
link = article_service.extract_url(body)
|
||
if link:
|
||
return await _collect_article(session, channel, link)
|
||
body, tags = split_tags(body)
|
||
title, rest = split_title(body)
|
||
if not title:
|
||
return {"action": "ignored", "item_id": None, "reply": "内容是空的,没记。"}
|
||
creator = await creator_id(session, channel)
|
||
if kind == "knowledge":
|
||
item = await create_knowledge(
|
||
session,
|
||
ws_id=channel.workspace_id,
|
||
creator=creator,
|
||
title=title,
|
||
body=rest,
|
||
tags=tags,
|
||
)
|
||
return {
|
||
"action": "knowledge",
|
||
"item_id": item.id,
|
||
"reply": reply_text("knowledge", item),
|
||
}
|
||
item = await create_task(
|
||
session,
|
||
ws_id=channel.workspace_id,
|
||
creator=creator,
|
||
title=title,
|
||
body=rest,
|
||
tags=tags,
|
||
)
|
||
return {
|
||
"action": "task",
|
||
"item_id": item.id,
|
||
"reply": reply_text("task", item),
|
||
}
|