Init: 第一个版本

app/chunker.py

@@ -0,0 +1,46 @@
+import re
+
+from app.config import CHUNK_OVERLAP, CHUNK_SIZE
+
+
+def chunk_text(text: str, size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
+    """按中文标点和段落边界切分，相邻切片保留一小段重叠。"""
+    text = re.sub(r"\r\n?", "\n", text).replace("\x00", "").strip()
+    if not text:
+        return []
+    if len(text) <= size:
+        return [text]
+
+    units = [unit for unit in re.split(r"(?<=[。！？；\n])", text) if unit]
+    pieces: list[str] = []
+    step = max(1, size - overlap)
+    for unit in units:
+        if len(unit) <= size:
+            pieces.append(unit)
+            continue
+        for start in range(0, len(unit), step):
+            pieces.append(unit[start : start + size])
+
+    chunks: list[str] = []
+    buffer = ""
+    for piece in pieces:
+        if not buffer:
+            buffer = piece
+            continue
+        if len(buffer) + len(piece) <= size:
+            buffer += piece
+            continue
+        chunks.append(buffer.strip())
+        tail = buffer[-overlap:] if overlap else ""
+        candidate = f"{tail}{piece}"
+        buffer = candidate if len(candidate) <= size else piece
+    if buffer.strip():
+        chunks.append(buffer.strip())
+
+    result: list[str] = []
+    for chunk in chunks:
+        chunk = re.sub(r"\n{3,}", "\n\n", chunk).strip()
+        if len(chunk) < 8 or (result and chunk == result[-1]):
+            continue
+        result.append(chunk)
+    return result or [text]