Steven
626f83fb9c
CI / backend (push) Canceled after 0s
公众号的文章也能抓：走 mp.weixin.qq.com 的 DOM（#js_content），
按 data-src 取图，不靠通用抽取器——它会把公众号的图全丢掉。
正文里的外链图下载进我们自己的存储（最多 12 张、单张 8 MB），
地址改成 /attachments/{id}/raw?签名：公众号那边一换图，外链就是一排红叉。
抓取入口谁都能触发，所以只允许公网地址（内网段、回环、链路本地、
保留段、解析不出的域名一律拒绝），否则就是个 SSRF 跳板。
抓不到要说人话："这个链接没抓下来（原因）。想直接记成任务的话，
加「任务：」前缀再发一次。" 反爬验证页、SPA 空壳都走这条路。
- 新增 services/article.py：extract_url / parse / fetch / store_images / compose_body
- wechat_intake：整条消息是链接时走抓取，落知识草稿，回「已存成知识草稿：标题」
- 依赖 trafilatura==2.0.0（装法见 requirements.txt 注释：必须逐个装再 --no-deps）
- 测试 9 例：链接判定、公众号解析、验证页、通用页裁导航、SSRF 拒绝、图片本地化、失败提示
514 lines
18 KiB
Python
514 lines
18 KiB
Python
"""把一条链接抓成一篇知识。
|
||
|
||
微信里发一条链接，多半就是想"把这篇文章存下来"。这里负责三件事：
|
||
|
||
1. **取正文**：用 trafilatura 的正文抽取（开源社区里在中文站上表现最稳的一个），
|
||
微信公众号单独走一条路——它的正文在 `#js_content` 里、图片放在 `data-src`，
|
||
通用抽取器会把图丢掉。
|
||
2. **留来源**：正文开头记一行原文地址与作者，日后要核对时不用猜。
|
||
3. **图片转存**：文章里的图片不是躺在我们的存储里，公众号一改就变红叉。
|
||
所以抓下来的图逐张存成附件，正文里的地址换成我们自己的签名直链。
|
||
|
||
抓不下来不硬扛：调用方拿到 `ArticleError` 就把原链接当普通文本存成任务，
|
||
用户至少不会丢东西。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import ipaddress
|
||
import re
|
||
import socket
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
from urllib.parse import urlparse
|
||
|
||
import httpx
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
from starlette.concurrency import run_in_threadpool
|
||
|
||
# 先让 app.models 加载完：它在文件末尾反向导入各域的租户表清单，
|
||
# 直接 import models_attachment 会撞上"部分初始化"的循环导入。
|
||
from .. import models as _models # noqa: F401
|
||
from ..config import get_settings
|
||
from ..models_attachment import Attachment
|
||
from . import signing, storage
|
||
|
||
# 一条消息里只认第一个链接；带别的字就当普通任务（"看下 https://… 这个" 不该被吞成收藏）
|
||
URL_RE = re.compile(r"https?://[^\s<>\"'）)】。，；]+", re.I)
|
||
|
||
MAX_ARTICLES_BYTES = 8 * 1024 * 1024
|
||
MAX_IMAGES = 12
|
||
MAX_IMAGE_BYTES = 8 * 1024 * 1024
|
||
MAX_BODY_CHARS = 200_000
|
||
FETCH_TIMEOUT = 25.0
|
||
|
||
DESKTOP_UA = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
||
" (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
|
||
)
|
||
# 公众号对"非微信浏览器"会压缩内容、图片懒加载，用微信 UA 拿到的才是完整一篇。
|
||
WECHAT_UA = (
|
||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15"
|
||
" (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2e)"
|
||
" NetType/WIFI Language/zh_CN"
|
||
)
|
||
|
||
IMAGE_EXTS = {"jpeg": "jpg", "jpg": "jpg", "png": "png", "gif": "gif", "webp": "webp"}
|
||
|
||
|
||
class ArticleError(RuntimeError):
|
||
"""抓取/解析没成功。调用方据此决定降级方式。"""
|
||
|
||
|
||
@dataclass
|
||
class Article:
|
||
url: str
|
||
title: str
|
||
body_md: str
|
||
site: str = ""
|
||
author: str = ""
|
||
images: list[str] = field(default_factory=list)
|
||
|
||
|
||
def extract_url(text: str) -> str | None:
|
||
"""这一整句话就是个链接吗？
|
||
|
||
允许链接前后有点标点、空格（"看下：https://x" 不算）；带了别的内容就当普通文本，
|
||
免得用户随口提一句网址，心情是"记条任务"，结果被存成一篇文章。
|
||
"""
|
||
stripped = text.strip()
|
||
match = URL_RE.search(stripped)
|
||
if not match:
|
||
return None
|
||
rest = (stripped[: match.start()] + stripped[match.end() :]).strip()
|
||
rest = rest.strip(" \t\r\n:：，。；;、!！?？（）()【】[]<>「」")
|
||
if rest:
|
||
return None
|
||
return match.group(0)
|
||
|
||
|
||
def is_wechat_article(url: str) -> bool:
|
||
host = (urlparse(url).hostname or "").lower()
|
||
return host == "mp.weixin.qq.com"
|
||
|
||
|
||
async def _assert_public(url: str) -> None:
|
||
"""只抓公网地址。
|
||
|
||
这个入口能由工作区以外的人（微信里的任何人）触发，放任它就是一台 SSRF 跳板：
|
||
`http://127.0.0.1:8200/version`、`http://169.254.169.254/` 之类都能被打。
|
||
"""
|
||
parsed = urlparse(url)
|
||
if parsed.scheme not in ("http", "https"):
|
||
raise ArticleError("只支持 http/https 链接")
|
||
host = parsed.hostname or ""
|
||
if not host:
|
||
raise ArticleError("链接里没有主机名")
|
||
try:
|
||
infos = await _resolve(host)
|
||
except OSError as exc: # 解析不出来就别猜
|
||
raise ArticleError(f"解析不了这个域名：{exc}") from exc
|
||
for _family, _type, _proto, _canon, sockaddr in infos:
|
||
ip = ipaddress.ip_address(sockaddr[0])
|
||
if (
|
||
ip.is_private
|
||
or ip.is_loopback
|
||
or ip.is_link_local
|
||
or ip.is_reserved
|
||
or ip.is_multicast
|
||
or ip.is_unspecified
|
||
):
|
||
raise ArticleError("这个地址在内网，不抓")
|
||
|
||
|
||
async def _resolve(host: str):
|
||
import asyncio
|
||
|
||
def lookup():
|
||
return socket.getaddrinfo(host, None)
|
||
|
||
return await asyncio.to_thread(lookup)
|
||
|
||
|
||
def _pick_ua(url: str) -> str:
|
||
return WECHAT_UA if is_wechat_article(url) else DESKTOP_UA
|
||
|
||
|
||
def _looks_blocked(html: str) -> bool:
|
||
probes = (
|
||
"环境异常",
|
||
"完成验证后即可继续访问",
|
||
"请在微信客户端打开链接",
|
||
"该内容已被发布者删除",
|
||
"此内容因违规无法查看",
|
||
)
|
||
return any(p in html for p in probes)
|
||
|
||
|
||
def _clean_text(value: str) -> str:
|
||
return re.sub(r"[ \t\u00a0]+", " ", (value or "").replace("\r", "")).strip()
|
||
|
||
|
||
def _wechat_meta(html: str) -> tuple[str, str, list[tuple[str, str]]]:
|
||
"""从公众号页面里取 (标题, 作者/公众号名, [(图片地址, alt)]) 与正文节点。"""
|
||
from lxml import html as lxml_html
|
||
|
||
tree = lxml_html.fromstring(html)
|
||
title = ""
|
||
for xpath in (
|
||
'//*[@id="activity-name"]//text()',
|
||
'//meta[@property="og:title"]/@content',
|
||
"//title/text()",
|
||
):
|
||
parts = [_clean_text(t) for t in tree.xpath(xpath)]
|
||
title = " ".join(p for p in parts if p)
|
||
if title:
|
||
break
|
||
author = ""
|
||
for xpath in (
|
||
'//*[@id="js_name"]//text()',
|
||
'//meta[@name="author"]/@content',
|
||
'//*[@id="meta_content"]//*[@class="rich_media_meta_text"]//text()',
|
||
):
|
||
parts = [_clean_text(t) for t in tree.xpath(xpath)]
|
||
author = " ".join(p for p in parts if p)
|
||
if author:
|
||
break
|
||
return title, author, tree
|
||
|
||
|
||
def _wechat_body(html: str) -> str:
|
||
"""`#js_content` → markdown。
|
||
|
||
公众号正文是一堆嵌套 `section` 拼版，段落文字常被拆成多个 `span`；
|
||
这里只取"没有块级子节点"的元素，避免同一个句子被输出好几遍。
|
||
"""
|
||
from lxml import html as lxml_html
|
||
|
||
tree = lxml_html.fromstring(html)
|
||
nodes = tree.xpath('//*[@id="js_content"]')
|
||
if not nodes:
|
||
return ""
|
||
content = nodes[0]
|
||
|
||
block_tags = {
|
||
"p",
|
||
"section",
|
||
"div",
|
||
"blockquote",
|
||
"li",
|
||
"h1",
|
||
"h2",
|
||
"h3",
|
||
"h4",
|
||
"h5",
|
||
"h6",
|
||
"pre",
|
||
}
|
||
lines: list[str] = []
|
||
for node in content.iter():
|
||
tag = node.tag if isinstance(node.tag, str) else ""
|
||
if tag == "img":
|
||
src = node.get("data-src") or node.get("src") or ""
|
||
if src.startswith("http"):
|
||
lines.append(f"")
|
||
continue
|
||
if tag == "br":
|
||
lines.append("")
|
||
continue
|
||
if tag not in block_tags:
|
||
continue
|
||
# 还有块级子孙的容器交给子节点输出，自己不出文字，否则整段重复
|
||
if any(
|
||
(child.tag if isinstance(child.tag, str) else "") in block_tags
|
||
for child in node.iterdescendants()
|
||
):
|
||
continue
|
||
text = _clean_text("".join(node.itertext()))
|
||
if not text:
|
||
continue
|
||
if tag.startswith("h") and len(tag) == 2:
|
||
text = f"## {text}"
|
||
elif tag == "li":
|
||
text = f"- {text}"
|
||
elif tag == "blockquote":
|
||
text = f"> {text}"
|
||
lines.append(text)
|
||
|
||
# 相邻空行压成一个，段间留空行
|
||
out: list[str] = []
|
||
for line in lines:
|
||
if not line and (not out or not out[-1]):
|
||
continue
|
||
out.append(line)
|
||
return "\n\n".join(s.strip() for s in "\n".join(out).split("\n\n") if s.strip())
|
||
|
||
|
||
def _block_markdown(html: str) -> str:
|
||
"""兜底的正文提取：找出"装着最多段文字"的那个容器，再按段落拼成 markdown。
|
||
|
||
为什么还要这一手：trafilatura 在结构清楚的资讯站上很准，但遇到"整页一个 div、
|
||
导航和正文混在一起"的老站点（政府站、企业站）时会连导航一起端出来。
|
||
这里用最笨但稳的判据——哪个容器里的 <p> 文字最多，正文就在哪个容器里。
|
||
"""
|
||
from lxml import html as lxml_html
|
||
|
||
tree = lxml_html.fromstring(html)
|
||
best = None
|
||
best_len = 0
|
||
for node in tree.iter("div", "article", "section", "main", "td"):
|
||
total = sum(len("".join(p.itertext()).strip()) for p in node.iter("p"))
|
||
if total > best_len:
|
||
best, best_len = node, total
|
||
if best is None or best_len < 200:
|
||
return ""
|
||
|
||
parts: list[str] = []
|
||
for node in best.iter():
|
||
tag = node.tag if isinstance(node.tag, str) else ""
|
||
if tag == "img":
|
||
src = node.get("data-src") or node.get("src") or ""
|
||
if src.startswith("http"):
|
||
parts.append(f"")
|
||
continue
|
||
if tag not in {"p", "h1", "h2", "h3", "h4", "blockquote", "li"}:
|
||
continue
|
||
text = _clean_text("".join(node.itertext()))
|
||
if not text:
|
||
continue
|
||
if tag.startswith("h") and len(tag) == 2:
|
||
text = f"## {text}"
|
||
elif tag == "li":
|
||
text = f"- {text}"
|
||
elif tag == "blockquote":
|
||
text = f"> {text}"
|
||
parts.append(text)
|
||
return "\n\n".join(parts).strip()
|
||
|
||
|
||
def _body_score(body: str) -> int:
|
||
"""给一份抽取结果打分：有多少段"像人写的段落"。
|
||
|
||
只用来在两个候选之间挑一个，不需要绝对准确——比较的是同一篇文章的两种切法。
|
||
"""
|
||
lines = [line.strip() for line in body.splitlines() if line.strip()]
|
||
return sum(1 for line in lines if len(line) >= 40)
|
||
|
||
|
||
def _trim_chrome(body: str) -> str:
|
||
"""去掉抽取器一并带出来的导航与页脚。
|
||
|
||
按行看，找出第一行和最后一行"像正文的"（够长、或者有句读），把两头那些短标签
|
||
（"Java教程"、"Sign In"）切掉。按行而不是按段落，是因为导航往往是一行一个、
|
||
中间不空行，按段落切会把它和正文粘成一整块，判不出边界。
|
||
"""
|
||
lines = body.splitlines()
|
||
if not lines:
|
||
return ""
|
||
|
||
def substantive(line: str) -> bool:
|
||
text = line.strip()
|
||
if not text or text.startswith(("![", "> ")):
|
||
return False
|
||
plain = text.lstrip("#-* ")
|
||
return len(plain) >= 20 or bool(re.search(r"[。！？；]", plain))
|
||
|
||
first = next((i for i, line in enumerate(lines) if substantive(line)), None)
|
||
if first is None:
|
||
return body.strip()
|
||
last = max(i for i, line in enumerate(lines) if substantive(line))
|
||
# 图片通常紧贴在正文前后，边界外紧邻的图保留
|
||
while first > 0 and lines[first - 1].strip().startswith("!["):
|
||
first -= 1
|
||
while last + 1 < len(lines) and lines[last + 1].strip().startswith("!["):
|
||
last += 1
|
||
return "\n".join(lines[first : last + 1]).strip()
|
||
|
||
|
||
def _generic_article(html: str, url: str) -> Article:
|
||
"""非公众号：交给 trafilatura，它在正文/噪声的判别上比手搓规则靠谱得多。
|
||
|
||
先按 `favor_precision` 抠最干净的一块；它经常因为"看着都不像正文"而返回空，
|
||
再退回默认召回模式。两头再削一刀导航。
|
||
"""
|
||
import trafilatura
|
||
|
||
metadata = trafilatura.extract_metadata(html, default_url=url)
|
||
common = {
|
||
"url": url,
|
||
"output_format": "markdown",
|
||
"include_images": True,
|
||
"include_links": True,
|
||
"include_formatting": True,
|
||
}
|
||
body = trafilatura.extract(html, favor_precision=True, **common) or ""
|
||
if len(body.strip()) < 120:
|
||
body = trafilatura.extract(html, **common) or ""
|
||
if not body.strip():
|
||
# 再退一步：trafilatura 认不出结构时，退到纯文本
|
||
body = trafilatura.extract(html, url=url, output_format="txt") or ""
|
||
body = _trim_chrome(body)
|
||
|
||
# 老站点（政府站、企业站）常把导航和正文塞进同一个 div，trafilatura 会连导航一起端出来。
|
||
# 用"哪个容器里段落文字最多"另取一份，两份按"像样段落的条数"挑更干净的那个。
|
||
block = _trim_chrome(_block_markdown(html))
|
||
if _body_score(block) > _body_score(body):
|
||
body = block
|
||
if not body.strip():
|
||
raise ArticleError("这个页面里没找到正文（可能是动态渲染或需要登录）")
|
||
title = _clean_text(getattr(metadata, "title", "") or "")
|
||
return Article(
|
||
url=url,
|
||
title=title,
|
||
body_md=body.strip(),
|
||
site=_clean_text(getattr(metadata, "sitename", "") or ""),
|
||
author=_clean_text(getattr(metadata, "author", "") or ""),
|
||
)
|
||
|
||
|
||
def parse_article(html: str, url: str) -> Article:
|
||
if _looks_blocked(html):
|
||
raise ArticleError("微信把这篇拦下来了（需要人工验证或链接已失效）")
|
||
if is_wechat_article(url):
|
||
title, author, _tree = _wechat_meta(html)
|
||
body = _wechat_body(html)
|
||
if not body.strip():
|
||
# 公众号偶尔会换结构，退回通用抽取，宁可少图也别丢正文
|
||
article = _generic_article(html, url)
|
||
article.author = author or article.author
|
||
return article
|
||
return Article(
|
||
url=url,
|
||
title=title,
|
||
body_md=body.strip(),
|
||
site="微信公众号",
|
||
author=author,
|
||
)
|
||
return _generic_article(html, url)
|
||
|
||
|
||
async def fetch_article(url: str) -> Article:
|
||
await _assert_public(url)
|
||
headers = {
|
||
"User-Agent": _pick_ua(url),
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.6",
|
||
}
|
||
try:
|
||
async with httpx.AsyncClient(
|
||
follow_redirects=True, timeout=FETCH_TIMEOUT, headers=headers
|
||
) as client:
|
||
resp = await client.get(url)
|
||
resp.raise_for_status()
|
||
raw = resp.content
|
||
except httpx.HTTPStatusError as exc:
|
||
raise ArticleError(f"对方站点返回 {exc.response.status_code}") from exc
|
||
except httpx.HTTPError as exc:
|
||
raise ArticleError(f"取不到这个页面：{type(exc).__name__}") from exc
|
||
if len(raw) > MAX_ARTICLES_BYTES:
|
||
raise ArticleError("页面太大了，不像是篇文章")
|
||
html = raw.decode(resp.encoding or "utf-8", errors="replace")
|
||
article = parse_article(html, url)
|
||
article.title = (article.title or urlparse(url).hostname or "未命名")[:180]
|
||
return article
|
||
|
||
|
||
async def _download_image(client: httpx.AsyncClient, url: str) -> tuple[bytes, str] | None:
|
||
try:
|
||
resp = await client.get(url)
|
||
resp.raise_for_status()
|
||
except httpx.HTTPError:
|
||
return None
|
||
if len(resp.content) > MAX_IMAGE_BYTES:
|
||
return None
|
||
ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
|
||
if not ctype.startswith("image/"):
|
||
# 公众号图片地址带 wx_fmt 参数，content-type 偶尔是 octet-stream，按扩展名兜一下
|
||
fmt = urlparse(url).query or ""
|
||
m = re.search(r"wx_fmt=(\w+)", fmt)
|
||
guess = IMAGE_EXTS.get((m.group(1) if m else "").lower(), "")
|
||
if not guess:
|
||
return None
|
||
ctype = f"image/{'jpeg' if guess == 'jpg' else guess}"
|
||
return resp.content, ctype
|
||
|
||
|
||
async def store_images(
|
||
session: AsyncSession,
|
||
*,
|
||
ws_id: uuid.UUID,
|
||
item_id: uuid.UUID,
|
||
uploader_id: uuid.UUID,
|
||
body_md: str,
|
||
limit: int = MAX_IMAGES,
|
||
) -> tuple[str, list[Attachment]]:
|
||
"""把正文里引用的远程图片存成我们自己的附件，返回改写后的正文与附件行。
|
||
|
||
失败就当没这张图：一张打不开的配图不该让整篇收藏失败。
|
||
"""
|
||
from io import BytesIO
|
||
|
||
urls = re.findall(r"!\[[^\]]*\]\((https?://[^)\s]+)\)", body_md)
|
||
unique: list[str] = []
|
||
for url in urls:
|
||
if url not in unique:
|
||
unique.append(url)
|
||
if len(unique) >= limit:
|
||
break
|
||
if not unique:
|
||
return body_md, []
|
||
|
||
settings = get_settings()
|
||
saved: list[Attachment] = []
|
||
mapping: dict[str, str] = {}
|
||
headers = {"User-Agent": DESKTOP_UA, "Referer": "https://mp.weixin.qq.com/"}
|
||
async with httpx.AsyncClient(
|
||
follow_redirects=True, timeout=FETCH_TIMEOUT, headers=headers
|
||
) as client:
|
||
for index, remote in enumerate(unique, start=1):
|
||
got = await _download_image(client, remote)
|
||
if got is None:
|
||
continue
|
||
payload, content_type = got
|
||
ext = IMAGE_EXTS.get(content_type.split("/")[-1], "jpg")
|
||
# 落盘是阻塞 IO：一篇公众号十几张图，别把事件循环按住
|
||
stored = await run_in_threadpool(
|
||
storage.save, BytesIO(payload), workspace_id=ws_id, item_id=item_id
|
||
)
|
||
if stored.size > settings.max_upload_mb * 1024 * 1024:
|
||
storage.remove(stored.key)
|
||
continue
|
||
row = Attachment(
|
||
workspace_id=ws_id,
|
||
item_id=item_id,
|
||
uploader_id=uploader_id,
|
||
filename=f"图{index}.{ext}",
|
||
content_type=content_type[:120],
|
||
size_bytes=stored.size,
|
||
sha256=stored.sha256,
|
||
storage_key=stored.key,
|
||
)
|
||
session.add(row)
|
||
await session.flush()
|
||
mapping[remote] = (
|
||
f"{settings.api_prefix}/workspaces/{ws_id}"
|
||
f"/attachments/{row.id}/raw?{signing.sign(ws_id, row.id)}"
|
||
)
|
||
saved.append(row)
|
||
|
||
for remote, local in mapping.items():
|
||
body_md = body_md.replace(remote, local)
|
||
return body_md, saved
|
||
|
||
|
||
def compose_body(article: Article) -> str:
|
||
"""正文 = 来源行 + 原文。来源要留在最上面，日后核对不用翻聊天记录。"""
|
||
bits = [f"> 原文：<{article.url}>"]
|
||
if article.site or article.author:
|
||
who = " · ".join(x for x in (article.site, article.author) if x)
|
||
bits[0] += f"（{who}）"
|
||
body = "\n".join(bits) + "\n\n" + article.body_md
|
||
if len(body) > MAX_BODY_CHARS:
|
||
body = body[:MAX_BODY_CHARS] + "\n\n…（原文太长，后面的内容没有保存）"
|
||
return body
|