47 lines
1.4 KiB
Python
47 lines
1.4 KiB
Python
import re
|
|
|
|
from app.config import CHUNK_OVERLAP, CHUNK_SIZE
|
|
|
|
|
|
def chunk_text(text: str, size: int = CHUNK_SIZE, overlap: int = CHUNK_OVERLAP) -> list[str]:
|
|
"""按中文标点和段落边界切分，相邻切片保留一小段重叠。"""
|
|
text = re.sub(r"\r\n?", "\n", text).replace("\x00", "").strip()
|
|
if not text:
|
|
return []
|
|
if len(text) <= size:
|
|
return [text]
|
|
|
|
units = [unit for unit in re.split(r"(?<=[。！？；\n])", text) if unit]
|
|
pieces: list[str] = []
|
|
step = max(1, size - overlap)
|
|
for unit in units:
|
|
if len(unit) <= size:
|
|
pieces.append(unit)
|
|
continue
|
|
for start in range(0, len(unit), step):
|
|
pieces.append(unit[start : start + size])
|
|
|
|
chunks: list[str] = []
|
|
buffer = ""
|
|
for piece in pieces:
|
|
if not buffer:
|
|
buffer = piece
|
|
continue
|
|
if len(buffer) + len(piece) <= size:
|
|
buffer += piece
|
|
continue
|
|
chunks.append(buffer.strip())
|
|
tail = buffer[-overlap:] if overlap else ""
|
|
candidate = f"{tail}{piece}"
|
|
buffer = candidate if len(candidate) <= size else piece
|
|
if buffer.strip():
|
|
chunks.append(buffer.strip())
|
|
|
|
result: list[str] = []
|
|
for chunk in chunks:
|
|
chunk = re.sub(r"\n{3,}", "\n\n", chunk).strip()
|
|
if len(chunk) < 8 or (result and chunk == result[-1]):
|
|
continue
|
|
result.append(chunk)
|
|
return result or [text]
|