added RAG, Multiuser, TG bot
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def chunk_text(text: str, *, chunk_size: int = 800, overlap: int = 120) -> list[str]:
|
||||
cleaned = (text or "").strip()
|
||||
if not cleaned:
|
||||
return []
|
||||
if len(cleaned) <= chunk_size:
|
||||
return [cleaned]
|
||||
chunks: list[str] = []
|
||||
start = 0
|
||||
while start < len(cleaned):
|
||||
end = min(len(cleaned), start + chunk_size)
|
||||
piece = cleaned[start:end].strip()
|
||||
if piece:
|
||||
chunks.append(piece)
|
||||
if end >= len(cleaned):
|
||||
break
|
||||
start = max(0, end - overlap)
|
||||
return chunks
|
||||
Reference in New Issue
Block a user