21 lines
601 B
Python
21 lines
601 B
Python
from __future__ import annotations
|
|
|
|
|
|
def chunk_text(text: str, *, chunk_size: int = 800, overlap: int = 120) -> list[str]:
|
|
cleaned = (text or "").strip()
|
|
if not cleaned:
|
|
return []
|
|
if len(cleaned) <= chunk_size:
|
|
return [cleaned]
|
|
chunks: list[str] = []
|
|
start = 0
|
|
while start < len(cleaned):
|
|
end = min(len(cleaned), start + chunk_size)
|
|
piece = cleaned[start:end].strip()
|
|
if piece:
|
|
chunks.append(piece)
|
|
if end >= len(cleaned):
|
|
break
|
|
start = max(0, end - overlap)
|
|
return chunks
|