用户发来 https://mp.weixin.qq.com/s/ZEOsi4R4JcmZ_mhS6qpcww,存进库的正文只有 67 个字符 ("原文链接 + 一个句号")。复盘发现公众号换了渲染方式:新版页面把整篇塞进 window.cgiDataNew 这个 JS 对象由前端渲染,页面里根本没有 id="js_content"; 三种 UA 拿到的字节数完全一样,也不是 UA 的问题。 - 从 JS 字符串字面量里读正文(content_noencode / desc),按 \xNN、\uNNNN 反转义, 话题标签是双重转义过的 HTML,先 unescape 再走 DOM。 - 配图字段排在文字之后,所以前后各找 2500 字符合图;剩下没用过的 mmbiz 图按 12 张上限补在末尾(图文型文章靠就近配对会漏)。 - 解析顺序:旧 DOM → 新版 JS 数据 → 通用抽取,一层层退。 实测两篇:2373 字符 + 12 张图、2772 字符 + 13 张图,作者也都取到了。 同时补上用户提的两件: - 知识能删了:DELETE /knowledge/{id}?confirm=true,软删 + 真删检索切片, 只有作者或 owner/admin 能删;列表卡片(悬停出现)和阅读页动作行各一个入口。 删完详情也回 7001,否则收藏的链接还能打开一篇已删除的知识。 - 分类下面有标签清单了:GET /knowledge/tags 按标签数"有多少条知识用了它", 侧栏渲染成 #标签 + 条数,点一下筛选、再点取消。
This commit is contained in:
@@ -15,6 +15,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html as html_mod
|
||||
import ipaddress
|
||||
import re
|
||||
import socket
|
||||
@@ -55,6 +56,15 @@ WECHAT_UA = (
|
||||
|
||||
IMAGE_EXTS = {"jpeg": "jpg", "jpg": "jpg", "png": "png", "gif": "gif", "webp": "webp"}
|
||||
|
||||
# 公众号新版页面(2024 之后)不再把正文放进 `#js_content`,而是把整篇塞进
|
||||
# `window.cgiDataNew` 这个 JS 对象里,由前端渲染。所以还得会读 JS 字符串字面量:
|
||||
# 换行写成 `\x0a`,话题链接是 `\x26lt;a …\x26gt;`(先转义成实体、再转义成 JS)。
|
||||
_WECHAT_FIELD_RE = re.compile(r"(?:content_noencode|desc)\s*:\s*'")
|
||||
_WECHAT_IMAGE_RE = re.compile(
|
||||
r"""(?:cdn_url|img_url|pic_url)\s*:\s*['"]((?:https?:)?//[^'"]*mmbiz\.qpic\.cn[^'"]*)['"]"""
|
||||
)
|
||||
_JS_ESCAPE_RE = re.compile(r"\\(x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|.)", re.S)
|
||||
|
||||
|
||||
class ArticleError(RuntimeError):
|
||||
"""抓取/解析没成功。调用方据此决定降级方式。"""
|
||||
@@ -174,6 +184,10 @@ def _wechat_meta(html: str) -> tuple[str, str, list[tuple[str, str]]]:
|
||||
author = " ".join(p for p in parts if p)
|
||||
if author:
|
||||
break
|
||||
if not author:
|
||||
# 新版页面没有 #js_name,公众号名在 cgiDataNew 的 nick_name 字段里
|
||||
found = re.search(r"nick_name\s*:\s*'([^']{1,80})'", html)
|
||||
author = _clean_text(found.group(1)) if found else ""
|
||||
return title, author, tree
|
||||
|
||||
|
||||
@@ -244,6 +258,115 @@ def _wechat_body(html: str) -> str:
|
||||
return "\n\n".join(s.strip() for s in "\n".join(out).split("\n\n") if s.strip())
|
||||
|
||||
|
||||
def _js_literal(text: str, quote_pos: int) -> str:
|
||||
"""读一个单引号 JS 字符串的原始内容(`quote_pos` 指向那个开引号)。"""
|
||||
out: list[str] = []
|
||||
i = quote_pos + 1
|
||||
while i < len(text):
|
||||
ch = text[i]
|
||||
if ch == "\\":
|
||||
out.append(text[i : i + 2])
|
||||
i += 2
|
||||
continue
|
||||
if ch == "'":
|
||||
break
|
||||
out.append(ch)
|
||||
i += 1
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def _unescape_js(raw: str) -> str:
|
||||
def repl(match: re.Match[str]) -> str:
|
||||
seq = match.group(1)
|
||||
if seq[0] in "xu" and len(seq) > 1:
|
||||
return chr(int(seq[1:], 16))
|
||||
return {"n": "\n", "t": "\t", "r": "\r", "'": "'", '"': '"', "\\": "\\"}.get(seq, seq)
|
||||
|
||||
return _JS_ESCAPE_RE.sub(repl, raw)
|
||||
|
||||
|
||||
def _wechat_data_body(html_text: str) -> str:
|
||||
"""新版公众号:从 `window.cgiDataNew` 的 JS 字段里把正文和配图捞出来。
|
||||
|
||||
一篇文章是若干「图片 + 一段说明文字」拼起来的,JS 里是
|
||||
`cdn_url: '…'` 后面紧跟 `desc` / `content_noencode`,所以按出现顺序取文字,
|
||||
再往回找它自己那张图,配图就跟着段落走。
|
||||
"""
|
||||
blocks: list[str] = []
|
||||
seen_text: set[str] = set()
|
||||
used_images: set[str] = set()
|
||||
for match in _WECHAT_FIELD_RE.finditer(html_text):
|
||||
piece = _fragment_to_markdown(_unescape_js(_js_literal(html_text, match.end() - 1)))
|
||||
if len(piece) < 30 or piece in seen_text:
|
||||
continue
|
||||
seen_text.add(piece)
|
||||
image = _nearby_image(html_text, match.start(), used_images)
|
||||
if image and "
|
||||
piece = f"{piece}\n\n"
|
||||
blocks.append(piece)
|
||||
# 图文型文章("小绿书")的图不在文字附近,靠上面的就近配对会漏掉;
|
||||
# 剩下的正文图统一补在末尾,宁可顺序差点也别丢图。
|
||||
leftover: list[str] = []
|
||||
for match in _WECHAT_IMAGE_RE.finditer(html_text):
|
||||
image = match.group(1)
|
||||
image = image if image.startswith("http") else f"https:{image}"
|
||||
if image in used_images:
|
||||
continue
|
||||
used_images.add(image)
|
||||
leftover.append(f"")
|
||||
if len(leftover) >= MAX_IMAGES:
|
||||
break
|
||||
if leftover:
|
||||
blocks.append("\n\n".join(leftover))
|
||||
return "\n\n".join(blocks).strip()
|
||||
|
||||
|
||||
def _fragment_to_markdown(fragment: str) -> str:
|
||||
"""把「实体转义 + 夹着标签」的正文片段变成 markdown 文本。
|
||||
|
||||
公众号把话题标签写成 `<a class="wx_topic_link" …>#调料</a>`,
|
||||
先 unescape 再当 HTML 处理,标签会被去掉、只留文字;`<img>` 转成 markdown。
|
||||
"""
|
||||
from lxml import html as lxml_html
|
||||
|
||||
fragment = html_mod.unescape(fragment or "")
|
||||
fragment = re.sub(
|
||||
r"""<img[^>]*?(?:data-src|src)\s*=\s*["']([^"']+)["'][^>]*>""",
|
||||
lambda m: f"\n\n})\n\n",
|
||||
fragment,
|
||||
flags=re.I,
|
||||
)
|
||||
fragment = re.sub(r"<br\s*/?>", "\n", fragment, flags=re.I)
|
||||
try:
|
||||
tree = lxml_html.fragment_fromstring(fragment, create_parent="div")
|
||||
text = tree.text_content()
|
||||
except Exception: # 片段结构烂到 lxml 都不认,直接剥标签
|
||||
text = re.sub(r"<[^>]+>", "", fragment)
|
||||
text = text.replace("\u00a0", " ")
|
||||
lines = [re.sub(r"[ \t\u3000]+", " ", line).strip() for line in text.splitlines()]
|
||||
out: list[str] = []
|
||||
for line in lines:
|
||||
if not line and (not out or not out[-1]):
|
||||
continue
|
||||
out.append(line)
|
||||
return "\n".join(out).strip()
|
||||
|
||||
|
||||
def _nearby_image(html_text: str, pos: int, used: set[str]) -> str | None:
|
||||
"""这条文字对应的配图:先往后找(新版页面图片字段排在文字之后),再往回找。"""
|
||||
for start, end in ((pos, pos + 2500), (max(0, pos - 2500), pos)):
|
||||
found = [
|
||||
m.group(1)
|
||||
for m in _WECHAT_IMAGE_RE.finditer(html_text, start, min(end, len(html_text)))
|
||||
if m.group(1) not in used
|
||||
]
|
||||
if found:
|
||||
image = found[0]
|
||||
return image if image.startswith("http") else f"https:{image}"
|
||||
return None
|
||||
|
||||
|
||||
def _block_markdown(html: str) -> str:
|
||||
"""兜底的正文提取:找出"装着最多段文字"的那个容器,再按段落拼成 markdown。
|
||||
|
||||
@@ -371,7 +494,7 @@ def parse_article(html: str, url: str) -> Article:
|
||||
raise ArticleError("微信把这篇拦下来了(需要人工验证或链接已失效)")
|
||||
if is_wechat_article(url):
|
||||
title, author, _tree = _wechat_meta(html)
|
||||
body = _wechat_body(html)
|
||||
body = _wechat_body(html) or _wechat_data_body(html)
|
||||
if not body.strip():
|
||||
# 公众号偶尔会换结构,退回通用抽取,宁可少图也别丢正文
|
||||
article = _generic_article(html, url)
|
||||
|
||||
Reference in New Issue
Block a user