Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 3 additions & 14 deletions lib/crewai-tools/src/crewai_tools/rag/chunkers/base_chunker.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ def _split_by_characters(self, text: str) -> list[str]:

def _merge_splits(self, splits: list[str], separator: str) -> list[str]:
"""Merge splits into chunks with proper overlap."""
separator = "" if self._keep_separator else separator
docs: list[str] = []
current_doc: list[str] = []
total = 0
Expand All @@ -115,13 +116,7 @@ def _merge_splits(self, splits: list[str], separator: str) -> list[str]:
split_len = len(split)

if total + split_len > self._chunk_size and current_doc:
if separator == "":
doc = "".join(current_doc)
else:
if self._keep_separator and separator == " ":
doc = "".join(current_doc)
else:
doc = separator.join(current_doc)
doc = separator.join(current_doc)

if doc:
docs.append(doc)
Expand All @@ -139,13 +134,7 @@ def _merge_splits(self, splits: list[str], separator: str) -> list[str]:
total += len(separator)

if current_doc:
if separator == "":
doc = "".join(current_doc)
else:
if self._keep_separator and separator == " ":
doc = "".join(current_doc)
else:
doc = separator.join(current_doc)
doc = separator.join(current_doc)

if doc:
docs.append(doc)
Expand Down
30 changes: 30 additions & 0 deletions lib/crewai-tools/tests/rag/test_data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,41 @@

import pytest

from crewai_tools.rag.chunkers.base_chunker import RecursiveCharacterTextSplitter
from crewai_tools.rag.chunkers.structured_chunker import JsonChunker
from crewai_tools.rag.data_types import DataType, DataTypes
from crewai_tools.rag.loaders.jsonl_loader import JSONLLoader


class TestDataTypeChunkers:
@pytest.mark.parametrize(
("data_type", "text"),
[
(DataType.TEXT, "a\nb"),
(DataType.TEXT, "Para one.\n\nPara two."),
(DataType.JSONL, '{"a": 1}\n{"b": 2}'),
],
)
def test_kept_separators_are_not_duplicated(
self, data_type: DataType, text: str
) -> None:
assert data_type.get_chunker().chunk(text) == [text]

def test_kept_separators_at_chunk_boundary(self) -> None:
splitter = RecursiveCharacterTextSplitter(chunk_size=5, chunk_overlap=0)

assert splitter.split_text("a\nb\nc\nd") == ["a\nb\nc", "\nc\nd"]

@pytest.mark.parametrize("separator", ["\n", "\n\n", ", ", " "])
def test_removed_separators_are_reinserted(self, separator: str) -> None:
splitter = RecursiveCharacterTextSplitter(
separators=[separator], keep_separator=False
)
text = separator.join(["a", "b", "c"])

assert splitter.split_text(text) == [text]


class TestDataTypesFromContentGitHub:
"""GitHub URL detection must use hostname matching, not substrings."""

Expand Down
Loading