"""把一条链接抓成一篇知识。

微信里发一条链接，多半就是想"把这篇文章存下来"。这里负责三件事：

1. **取正文**：用 trafilatura 的正文抽取（开源社区里在中文站上表现最稳的一个），
   微信公众号单独走一条路——它的正文在 `#js_content` 里、图片放在 `data-src`，
   通用抽取器会把图丢掉。
2. **留来源**：正文开头记一行原文地址与作者，日后要核对时不用猜。
3. **图片转存**：文章里的图片不是躺在我们的存储里，公众号一改就变红叉。
   所以抓下来的图逐张存成附件，正文里的地址换成我们自己的签名直链。

抓不下来不硬扛：调用方拿到 `ArticleError` 就把原链接当普通文本存成任务，
用户至少不会丢东西。
"""

from __future__ import annotations

import html as html_mod
import ipaddress
import re
import socket
import uuid
from dataclasses import dataclass, field
from urllib.parse import urlparse

import httpx
from sqlalchemy.ext.asyncio import AsyncSession
from starlette.concurrency import run_in_threadpool

# 先让 app.models 加载完：它在文件末尾反向导入各域的租户表清单，
# 直接 import models_attachment 会撞上"部分初始化"的循环导入。
from .. import models as _models  # noqa: F401
from ..config import get_settings
from ..models_attachment import Attachment
from . import signing, storage

# 一条消息里只认第一个链接；带别的字就当普通任务（"看下 https://… 这个" 不该被吞成收藏）
URL_RE = re.compile(r"https?://[^\s<>\"'）)】。，；]+", re.I)

MAX_ARTICLES_BYTES = 8 * 1024 * 1024
MAX_IMAGES = 12
MAX_IMAGE_BYTES = 8 * 1024 * 1024
MAX_BODY_CHARS = 200_000
FETCH_TIMEOUT = 25.0

DESKTOP_UA = (
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
    " (KHTML, like Gecko) Chrome/126.0 Safari/537.36"
)
# 公众号对"非微信浏览器"会压缩内容、图片懒加载，用微信 UA 拿到的才是完整一篇。
WECHAT_UA = (
    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15"
    " (KHTML, like Gecko) Mobile/15E148 MicroMessenger/8.0.44(0x18002c2e)"
    " NetType/WIFI Language/zh_CN"
)

IMAGE_EXTS = {"jpeg": "jpg", "jpg": "jpg", "png": "png", "gif": "gif", "webp": "webp"}

# 公众号新版页面（2024 之后）不再把正文放进 `#js_content`，而是把整篇塞进
# `window.cgiDataNew` 这个 JS 对象里，由前端渲染。所以还得会读 JS 字符串字面量：
# 换行写成 `\x0a`，话题链接是 `\x26lt;a …\x26gt;`（先转义成实体、再转义成 JS）。
_WECHAT_FIELD_RE = re.compile(r"(?:content_noencode|desc)\s*:\s*'")
_WECHAT_IMAGE_RE = re.compile(
    r"""(?:cdn_url|img_url|pic_url)\s*:\s*['"]((?:https?:)?//[^'"]*mmbiz\.qpic\.cn[^'"]*)['"]"""
)
_JS_ESCAPE_RE = re.compile(r"\\(x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|.)", re.S)


class ArticleError(RuntimeError):
    """抓取/解析没成功。调用方据此决定降级方式。"""


@dataclass
class Article:
    url: str
    title: str
    body_md: str
    site: str = ""
    author: str = ""
    images: list[str] = field(default_factory=list)


def extract_url(text: str) -> str | None:
    """这一整句话就是个链接吗？

    允许链接前后有点标点、空格（"看下：https://x" 不算）；带了别的内容就当普通文本，
    免得用户随口提一句网址，心情是"记条任务"，结果被存成一篇文章。
    """
    stripped = text.strip()
    match = URL_RE.search(stripped)
    if not match:
        return None
    rest = (stripped[: match.start()] + stripped[match.end() :]).strip()
    rest = rest.strip(" \t\r\n:：，。；;、!！?？（）()【】[]<>「」")
    if rest:
        return None
    return match.group(0)


def is_wechat_article(url: str) -> bool:
    host = (urlparse(url).hostname or "").lower()
    return host == "mp.weixin.qq.com"


async def _assert_public(url: str) -> None:
    """只抓公网地址。

    这个入口能由工作区以外的人（微信里的任何人）触发，放任它就是一台 SSRF 跳板：
    `http://127.0.0.1:8200/version`、`http://169.254.169.254/` 之类都能被打。
    """
    parsed = urlparse(url)
    if parsed.scheme not in ("http", "https"):
        raise ArticleError("只支持 http/https 链接")
    host = parsed.hostname or ""
    if not host:
        raise ArticleError("链接里没有主机名")
    try:
        infos = await _resolve(host)
    except OSError as exc:  # 解析不出来就别猜
        raise ArticleError(f"解析不了这个域名：{exc}") from exc
    for _family, _type, _proto, _canon, sockaddr in infos:
        ip = ipaddress.ip_address(sockaddr[0])
        if (
            ip.is_private
            or ip.is_loopback
            or ip.is_link_local
            or ip.is_reserved
            or ip.is_multicast
            or ip.is_unspecified
        ):
            raise ArticleError("这个地址在内网，不抓")


async def _resolve(host: str):
    import asyncio

    def lookup():
        return socket.getaddrinfo(host, None)

    return await asyncio.to_thread(lookup)


def _pick_ua(url: str) -> str:
    return WECHAT_UA if is_wechat_article(url) else DESKTOP_UA


def _looks_blocked(html: str) -> bool:
    probes = (
        "环境异常",
        "完成验证后即可继续访问",
        "请在微信客户端打开链接",
        "该内容已被发布者删除",
        "此内容因违规无法查看",
    )
    return any(p in html for p in probes)


def _clean_text(value: str) -> str:
    return re.sub(r"[ \t\u00a0]+", " ", (value or "").replace("\r", "")).strip()


def _wechat_meta(html: str) -> tuple[str, str, list[tuple[str, str]]]:
    """从公众号页面里取 (标题, 作者/公众号名, [(图片地址, alt)]) 与正文节点。"""
    from lxml import html as lxml_html

    tree = lxml_html.fromstring(html)
    title = ""
    for xpath in (
        '//*[@id="activity-name"]//text()',
        '//meta[@property="og:title"]/@content',
        "//title/text()",
    ):
        parts = [_clean_text(t) for t in tree.xpath(xpath)]
        title = " ".join(p for p in parts if p)
        if title:
            break
    author = ""
    for xpath in (
        '//*[@id="js_name"]//text()',
        '//meta[@name="author"]/@content',
        '//*[@id="meta_content"]//*[@class="rich_media_meta_text"]//text()',
    ):
        parts = [_clean_text(t) for t in tree.xpath(xpath)]
        author = " ".join(p for p in parts if p)
        if author:
            break
    if not author:
        # 新版页面没有 #js_name，公众号名在 cgiDataNew 的 nick_name 字段里
        found = re.search(r"nick_name\s*:\s*'([^']{1,80})'", html)
        author = _clean_text(found.group(1)) if found else ""
    return title, author, tree


def _wechat_body(html: str) -> str:
    """`#js_content` → markdown。

    公众号正文是一堆嵌套 `section` 拼版，段落文字常被拆成多个 `span`；
    这里只取"没有块级子节点"的元素，避免同一个句子被输出好几遍。
    """
    from lxml import html as lxml_html

    tree = lxml_html.fromstring(html)
    nodes = tree.xpath('//*[@id="js_content"]')
    if not nodes:
        return ""
    content = nodes[0]

    block_tags = {
        "p",
        "section",
        "div",
        "blockquote",
        "li",
        "h1",
        "h2",
        "h3",
        "h4",
        "h5",
        "h6",
        "pre",
    }
    lines: list[str] = []
    for node in content.iter():
        tag = node.tag if isinstance(node.tag, str) else ""
        if tag == "img":
            src = node.get("data-src") or node.get("src") or ""
            if src.startswith("http"):
                lines.append(f"![]({src})")
            continue
        if tag == "br":
            lines.append("")
            continue
        if tag not in block_tags:
            continue
        # 还有块级子孙的容器交给子节点输出，自己不出文字，否则整段重复
        if any(
            (child.tag if isinstance(child.tag, str) else "") in block_tags
            for child in node.iterdescendants()
        ):
            continue
        text = _clean_text("".join(node.itertext()))
        if not text:
            continue
        if tag.startswith("h") and len(tag) == 2:
            text = f"## {text}"
        elif tag == "li":
            text = f"- {text}"
        elif tag == "blockquote":
            text = f"> {text}"
        lines.append(text)

    # 相邻空行压成一个，段间留空行
    out: list[str] = []
    for line in lines:
        if not line and (not out or not out[-1]):
            continue
        out.append(line)
    return "\n\n".join(s.strip() for s in "\n".join(out).split("\n\n") if s.strip())


def _js_literal(text: str, quote_pos: int) -> str:
    """读一个单引号 JS 字符串的原始内容（`quote_pos` 指向那个开引号）。"""
    out: list[str] = []
    i = quote_pos + 1
    while i < len(text):
        ch = text[i]
        if ch == "\\":
            out.append(text[i : i + 2])
            i += 2
            continue
        if ch == "'":
            break
        out.append(ch)
        i += 1
    return "".join(out)


def _unescape_js(raw: str) -> str:
    def repl(match: re.Match[str]) -> str:
        seq = match.group(1)
        if seq[0] in "xu" and len(seq) > 1:
            return chr(int(seq[1:], 16))
        return {"n": "\n", "t": "\t", "r": "\r", "'": "'", '"': '"', "\\": "\\"}.get(seq, seq)

    return _JS_ESCAPE_RE.sub(repl, raw)


def _wechat_data_body(html_text: str) -> str:
    """新版公众号：从 `window.cgiDataNew` 的 JS 字段里把正文和配图捞出来。

    一篇文章是若干「图片 + 一段说明文字」拼起来的，JS 里是
    `cdn_url: '…'` 后面紧跟 `desc` / `content_noencode`，所以按出现顺序取文字，
    再往回找它自己那张图，配图就跟着段落走。
    """
    blocks: list[str] = []
    seen_text: set[str] = set()
    used_images: set[str] = set()
    for match in _WECHAT_FIELD_RE.finditer(html_text):
        piece = _fragment_to_markdown(_unescape_js(_js_literal(html_text, match.end() - 1)))
        if len(piece) < 30 or piece in seen_text:
            continue
        seen_text.add(piece)
        image = _nearby_image(html_text, match.start(), used_images)
        if image and "![](" not in piece:
            used_images.add(image)
            piece = f"{piece}\n\n![]({image})"
        blocks.append(piece)
    # 图文型文章（"小绿书"）的图不在文字附近，靠上面的就近配对会漏掉；
    # 剩下的正文图统一补在末尾，宁可顺序差点也别丢图。
    leftover: list[str] = []
    for match in _WECHAT_IMAGE_RE.finditer(html_text):
        image = match.group(1)
        image = image if image.startswith("http") else f"https:{image}"
        if image in used_images:
            continue
        used_images.add(image)
        leftover.append(f"![]({image})")
        if len(leftover) >= MAX_IMAGES:
            break
    if leftover:
        blocks.append("\n\n".join(leftover))
    return "\n\n".join(blocks).strip()


def _fragment_to_markdown(fragment: str) -> str:
    """把「实体转义 + 夹着标签」的正文片段变成 markdown 文本。

    公众号把话题标签写成 `&lt;a class="wx_topic_link" …&gt;#调料&lt;/a&gt;`，
    先 unescape 再当 HTML 处理，标签会被去掉、只留文字；`<img>` 转成 markdown。
    """
    from lxml import html as lxml_html

    fragment = html_mod.unescape(fragment or "")
    fragment = re.sub(
        r"""<img[^>]*?(?:data-src|src)\s*=\s*["']([^"']+)["'][^>]*>""",
        lambda m: f"\n\n![]({m.group(1)})\n\n",
        fragment,
        flags=re.I,
    )
    fragment = re.sub(r"<br\s*/?>", "\n", fragment, flags=re.I)
    try:
        tree = lxml_html.fragment_fromstring(fragment, create_parent="div")
        text = tree.text_content()
    except Exception:  # 片段结构烂到 lxml 都不认，直接剥标签
        text = re.sub(r"<[^>]+>", "", fragment)
    text = text.replace("\u00a0", " ")
    lines = [re.sub(r"[ \t\u3000]+", " ", line).strip() for line in text.splitlines()]
    out: list[str] = []
    for line in lines:
        if not line and (not out or not out[-1]):
            continue
        out.append(line)
    return "\n".join(out).strip()


def _nearby_image(html_text: str, pos: int, used: set[str]) -> str | None:
    """这条文字对应的配图：先往后找（新版页面图片字段排在文字之后），再往回找。"""
    for start, end in ((pos, pos + 2500), (max(0, pos - 2500), pos)):
        found = [
            m.group(1)
            for m in _WECHAT_IMAGE_RE.finditer(html_text, start, min(end, len(html_text)))
            if m.group(1) not in used
        ]
        if found:
            image = found[0]
            return image if image.startswith("http") else f"https:{image}"
    return None


def _block_markdown(html: str) -> str:
    """兜底的正文提取：找出"装着最多段文字"的那个容器，再按段落拼成 markdown。

    为什么还要这一手：trafilatura 在结构清楚的资讯站上很准，但遇到"整页一个 div、
    导航和正文混在一起"的老站点（政府站、企业站）时会连导航一起端出来。
    这里用最笨但稳的判据——哪个容器里的 <p> 文字最多，正文就在哪个容器里。
    """
    from lxml import html as lxml_html

    tree = lxml_html.fromstring(html)
    best = None
    best_len = 0
    for node in tree.iter("div", "article", "section", "main", "td"):
        total = sum(len("".join(p.itertext()).strip()) for p in node.iter("p"))
        if total > best_len:
            best, best_len = node, total
    if best is None or best_len < 200:
        return ""

    parts: list[str] = []
    for node in best.iter():
        tag = node.tag if isinstance(node.tag, str) else ""
        if tag == "img":
            src = node.get("data-src") or node.get("src") or ""
            if src.startswith("http"):
                parts.append(f"![]({src})")
            continue
        if tag not in {"p", "h1", "h2", "h3", "h4", "blockquote", "li"}:
            continue
        text = _clean_text("".join(node.itertext()))
        if not text:
            continue
        if tag.startswith("h") and len(tag) == 2:
            text = f"## {text}"
        elif tag == "li":
            text = f"- {text}"
        elif tag == "blockquote":
            text = f"> {text}"
        parts.append(text)
    return "\n\n".join(parts).strip()


def _body_score(body: str) -> int:
    """给一份抽取结果打分：有多少段"像人写的段落"。

    只用来在两个候选之间挑一个，不需要绝对准确——比较的是同一篇文章的两种切法。
    """
    lines = [line.strip() for line in body.splitlines() if line.strip()]
    return sum(1 for line in lines if len(line) >= 40)


def _trim_chrome(body: str) -> str:
    """去掉抽取器一并带出来的导航与页脚。

    按行看，找出第一行和最后一行"像正文的"（够长、或者有句读），把两头那些短标签
    （"Java教程"、"Sign In"）切掉。按行而不是按段落，是因为导航往往是一行一个、
    中间不空行，按段落切会把它和正文粘成一整块，判不出边界。
    """
    lines = body.splitlines()
    if not lines:
        return ""

    def substantive(line: str) -> bool:
        text = line.strip()
        if not text or text.startswith(("![", "> ")):
            return False
        plain = text.lstrip("#-* 　")
        return len(plain) >= 20 or bool(re.search(r"[。！？；]", plain))

    first = next((i for i, line in enumerate(lines) if substantive(line)), None)
    if first is None:
        return body.strip()
    last = max(i for i, line in enumerate(lines) if substantive(line))
    # 图片通常紧贴在正文前后，边界外紧邻的图保留
    while first > 0 and lines[first - 1].strip().startswith("!["):
        first -= 1
    while last + 1 < len(lines) and lines[last + 1].strip().startswith("!["):
        last += 1
    return "\n".join(lines[first : last + 1]).strip()


def _generic_article(html: str, url: str) -> Article:
    """非公众号：交给 trafilatura，它在正文/噪声的判别上比手搓规则靠谱得多。

    先按 `favor_precision` 抠最干净的一块；它经常因为"看着都不像正文"而返回空，
    再退回默认召回模式。两头再削一刀导航。
    """
    import trafilatura

    metadata = trafilatura.extract_metadata(html, default_url=url)
    common = {
        "url": url,
        "output_format": "markdown",
        "include_images": True,
        "include_links": True,
        "include_formatting": True,
    }
    body = trafilatura.extract(html, favor_precision=True, **common) or ""
    if len(body.strip()) < 120:
        body = trafilatura.extract(html, **common) or ""
    if not body.strip():
        # 再退一步：trafilatura 认不出结构时，退到纯文本
        body = trafilatura.extract(html, url=url, output_format="txt") or ""
    body = _trim_chrome(body)

    # 老站点（政府站、企业站）常把导航和正文塞进同一个 div，trafilatura 会连导航一起端出来。
    # 用"哪个容器里段落文字最多"另取一份，两份按"像样段落的条数"挑更干净的那个。
    block = _trim_chrome(_block_markdown(html))
    if _body_score(block) > _body_score(body):
        body = block
    if not body.strip():
        raise ArticleError("这个页面里没找到正文（可能是动态渲染或需要登录）")
    title = _clean_text(getattr(metadata, "title", "") or "")
    return Article(
        url=url,
        title=title,
        body_md=body.strip(),
        site=_clean_text(getattr(metadata, "sitename", "") or ""),
        author=_clean_text(getattr(metadata, "author", "") or ""),
    )


def parse_article(html: str, url: str) -> Article:
    if _looks_blocked(html):
        raise ArticleError("微信把这篇拦下来了（需要人工验证或链接已失效）")
    if is_wechat_article(url):
        title, author, _tree = _wechat_meta(html)
        body = _wechat_body(html) or _wechat_data_body(html)
        if not body.strip():
            # 公众号偶尔会换结构，退回通用抽取，宁可少图也别丢正文
            article = _generic_article(html, url)
            article.author = author or article.author
            return article
        return Article(
            url=url,
            title=title,
            body_md=body.strip(),
            site="微信公众号",
            author=author,
        )
    return _generic_article(html, url)


async def fetch_article(url: str) -> Article:
    await _assert_public(url)
    headers = {
        "User-Agent": _pick_ua(url),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.6",
    }
    try:
        async with httpx.AsyncClient(
            follow_redirects=True, timeout=FETCH_TIMEOUT, headers=headers
        ) as client:
            resp = await client.get(url)
            resp.raise_for_status()
            raw = resp.content
    except httpx.HTTPStatusError as exc:
        raise ArticleError(f"对方站点返回 {exc.response.status_code}") from exc
    except httpx.HTTPError as exc:
        raise ArticleError(f"取不到这个页面：{type(exc).__name__}") from exc
    if len(raw) > MAX_ARTICLES_BYTES:
        raise ArticleError("页面太大了，不像是篇文章")
    html = raw.decode(resp.encoding or "utf-8", errors="replace")
    article = parse_article(html, url)
    article.title = (article.title or urlparse(url).hostname or "未命名")[:180]
    return article


async def _download_image(client: httpx.AsyncClient, url: str) -> tuple[bytes, str] | None:
    try:
        resp = await client.get(url)
        resp.raise_for_status()
    except httpx.HTTPError:
        return None
    if len(resp.content) > MAX_IMAGE_BYTES:
        return None
    ctype = (resp.headers.get("content-type") or "").split(";")[0].strip().lower()
    if not ctype.startswith("image/"):
        # 公众号图片地址带 wx_fmt 参数，content-type 偶尔是 octet-stream，按扩展名兜一下
        fmt = urlparse(url).query or ""
        m = re.search(r"wx_fmt=(\w+)", fmt)
        guess = IMAGE_EXTS.get((m.group(1) if m else "").lower(), "")
        if not guess:
            return None
        ctype = f"image/{'jpeg' if guess == 'jpg' else guess}"
    return resp.content, ctype


async def store_images(
    session: AsyncSession,
    *,
    ws_id: uuid.UUID,
    item_id: uuid.UUID,
    uploader_id: uuid.UUID,
    body_md: str,
    limit: int = MAX_IMAGES,
) -> tuple[str, list[Attachment]]:
    """把正文里引用的远程图片存成我们自己的附件，返回改写后的正文与附件行。

    失败就当没这张图：一张打不开的配图不该让整篇收藏失败。
    """
    from io import BytesIO

    urls = re.findall(r"!\[[^\]]*\]\((https?://[^)\s]+)\)", body_md)
    unique: list[str] = []
    for url in urls:
        if url not in unique:
            unique.append(url)
        if len(unique) >= limit:
            break
    if not unique:
        return body_md, []

    settings = get_settings()
    saved: list[Attachment] = []
    mapping: dict[str, str] = {}
    headers = {"User-Agent": DESKTOP_UA, "Referer": "https://mp.weixin.qq.com/"}
    async with httpx.AsyncClient(
        follow_redirects=True, timeout=FETCH_TIMEOUT, headers=headers
    ) as client:
        for index, remote in enumerate(unique, start=1):
            got = await _download_image(client, remote)
            if got is None:
                continue
            payload, content_type = got
            ext = IMAGE_EXTS.get(content_type.split("/")[-1], "jpg")
            # 落盘是阻塞 IO：一篇公众号十几张图，别把事件循环按住
            stored = await run_in_threadpool(
                storage.save, BytesIO(payload), workspace_id=ws_id, item_id=item_id
            )
            if stored.size > settings.max_upload_mb * 1024 * 1024:
                storage.remove(stored.key)
                continue
            row = Attachment(
                workspace_id=ws_id,
                item_id=item_id,
                uploader_id=uploader_id,
                filename=f"图{index}.{ext}",
                content_type=content_type[:120],
                size_bytes=stored.size,
                sha256=stored.sha256,
                storage_key=stored.key,
            )
            session.add(row)
            await session.flush()
            mapping[remote] = (
                f"{settings.api_prefix}/workspaces/{ws_id}"
                f"/attachments/{row.id}/raw?{signing.sign(ws_id, row.id)}"
            )
            saved.append(row)

    for remote, local in mapping.items():
        body_md = body_md.replace(remote, local)
    return body_md, saved


def compose_body(article: Article) -> str:
    """正文 = 来源行 + 原文。来源要留在最上面，日后核对不用翻聊天记录。"""
    bits = [f"> 原文：<{article.url}>"]
    if article.site or article.author:
        who = " · ".join(x for x in (article.site, article.author) if x)
        bits[0] += f"（{who}）"
    body = "\n".join(bits) + "\n\n" + article.body_md
    if len(body) > MAX_BODY_CHARS:
        body = body[:MAX_BODY_CHARS] + "\n\n…（原文太长，后面的内容没有保存）"
    return body