Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions lib/crewai-tools/src/crewai_tools/rag/data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,13 @@ def get_file_type(path: str) -> DataType | None:
if dtype:
return dtype

if "docs" in url.netloc or ("docs" in url.path and url.scheme != "file"):
return DataType.DOCS_SITE
hostname = (url.hostname or "").lower()
if hostname == "github.com" or hostname.endswith(".github.com"):
is_github = hostname == "github.com" or hostname.endswith(".github.com")
if "docs" in url.netloc or (
"docs" in url.path and url.scheme != "file" and not is_github
):
return DataType.DOCS_SITE
if is_github:
return DataType.GITHUB

return DataType.WEBSITE
Expand Down
27 changes: 27 additions & 0 deletions lib/crewai-tools/tests/rag/test_data_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,33 @@ def test_github_in_path_is_not_github(self) -> None:
== DataType.WEBSITE
)

def test_repository_named_docs_is_github(self) -> None:
# A "docs" path segment on github.com is a repository name, not a DocsSite.
assert (
DataTypes.from_content("https://github.com/crewAIInc/docs")
== DataType.GITHUB
)

def test_github_docs_host_is_still_a_docs_site(self) -> None:
assert (
DataTypes.from_content("https://docs.github.com/en/rest")
== DataType.DOCS_SITE
)


class TestDataTypesDocsSite:
"""Docs detection keeps working for non-GitHub hosts."""

@pytest.mark.parametrize(
"url",
[
"https://example.com/docs/intro",
"https://docs.example.com/guide",
],
)
def test_docs_urls_are_docs_sites(self, url: str) -> None:
assert DataTypes.from_content(url) == DataType.DOCS_SITE


class TestDataTypesJSONL:
"""JSONL uses the existing registry for explicit selection and discovery."""
Expand Down
Loading