"""把 Markdown 正文切成适合检索的小片段(M3)。 切片的目的是"喂给检索",不是"给人读",所以三条规则优先于其它考虑: 1. **不切坏结构**:代码块整块保留,表格不拆行,标题跟着它下面的段落走 (单独一句"结论"离开上下文就没意义)。 2. **片内自洽**:每片尽量包含一个完整意思,超长才按句号切。 3. **相邻片有重叠**:跨段的话被切断时,前后片都能读到完整句子,命中后引用才不乱。 不引入分词器:中文另起 `zhparser` 要装扩展,而片段足够短时字符级检索已经够用。 """ from __future__ import annotations import re from dataclasses import dataclass HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") FENCE_RE = re.compile(r"^\s*```") SENTENCE_RE = re.compile(r"[^。!?!?;;\n]+[。!?!?;;]?") @dataclass(frozen=True) class ChunkDraft: heading: str | None content: str def _split_long(text: str, max_chars: int) -> list[str]: """把超长段落按句子切开;单句仍然超长就硬切。""" pieces: list[str] = [] buffer = "" for sentence in SENTENCE_RE.findall(text): if not sentence.strip(): continue if len(buffer) + len(sentence) <= max_chars: buffer += sentence continue if buffer.strip(): pieces.append(buffer.strip()) if len(sentence) <= max_chars: buffer = sentence else: for start in range(0, len(sentence), max_chars): pieces.append(sentence[start : start + max_chars].strip()) buffer = "" if buffer.strip(): pieces.append(buffer.strip()) return [p for p in pieces if p] def split_markdown( text: str, *, max_chars: int = 600, overlap: int = 80, ) -> list[ChunkDraft]: """返回按顺序排列的切片。 `overlap` 是相邻片之间共享的尾部字数;不想要重叠就传 0。 """ if not text or not text.strip(): return [] normalized = text.replace("\r\n", "\n").replace("\r", "\n") blocks: list[tuple[str | None, str]] = [] heading: str | None = None paragraph: list[str] = [] in_fence = False def flush() -> None: if paragraph: body = "\n".join(paragraph).strip() if body: blocks.append((heading, body)) paragraph.clear() for line in normalized.split("\n"): if FENCE_RE.match(line): in_fence = not in_fence paragraph.append(line) continue if not in_fence: match = HEADING_RE.match(line) if match: flush() heading = match.group(2).strip() continue if not line.strip(): flush() continue paragraph.append(line) flush() chunks: list[ChunkDraft] = [] buffer = "" buffer_heading: str | None = None def flush_buffer() -> None: nonlocal buffer, buffer_heading if buffer.strip(): chunks.append(ChunkDraft(heading=buffer_heading, content=buffer.strip())) buffer = "" for block_heading, body in blocks: parts = _split_long(body, max_chars) if len(body) > max_chars else [body] for part in parts: candidate = f"{buffer}\n\n{part}".strip() if buffer else part if len(candidate) <= max_chars: if not buffer: buffer_heading = block_heading buffer = candidate continue flush_buffer() # 新片开头带上上一片的尾巴,避免"半句话"被切断后两边都读不通 tail = "" if overlap > 0 and chunks: tail = chunks[-1].content[-overlap:] if "\n" in tail: tail = tail.split("\n", 1)[-1] buffer_heading = block_heading buffer = f"{tail}\n\n{part}".strip() if tail else part if len(buffer) > max_chars: # 带尾巴后仍然超长:放弃尾巴,保证不超限 buffer = part flush_buffer() return chunks