Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/preserve-decimal-sentences.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/rag": patch
---

Preserve decimal numbers and prices when splitting sentences. Refs #1542.
12 changes: 12 additions & 0 deletions packages/rag/src/__tests__/recursive-chunker.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,19 @@
import { describe, expect, it } from "vitest";
import { RecursiveChunker } from "../chunkers/recursive-chunker";
import { whitespaceTokenizer } from "../utils/tokenizer";

describe("RecursiveChunker", () => {
it("keeps decimals intact when recursively splitting a paragraph", () => {
const text = "Version 2.5 is out. It costs $3.99 today.";
const expected = ["Version 2.5 is out.", " It costs $3.99 today."];
const chunks = new RecursiveChunker(whitespaceTokenizer).chunk(text, { maxTokens: 4 });
expect(chunks.map((chunk) => chunk.content)).toEqual([
expected[0],
expected[0],
expected[1]?.trim(),
]);
});

it("splits long paragraphs then falls back to sentence/token chunking", () => {
const text = [
"Paragraph one short.",
Expand Down
18 changes: 18 additions & 0 deletions packages/rag/src/__tests__/semantic-chunker.spec.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,25 @@
import { describe, expect, it } from "vitest";
import { SemanticChunker } from "../chunkers/semantic-chunker";
import { whitespaceTokenizer } from "../utils/tokenizer";

describe("SemanticChunker", () => {
it("passes complete decimal sentences to the semantic embedder", async () => {
const text = "Version 2.5 is out. It costs $3.99 today.";
const expected = ["Version 2.5 is out.", " It costs $3.99 today."];
const inputs: string[] = [];
const chunks = await new SemanticChunker(whitespaceTokenizer).chunk(text, {
maxTokens: 4,
embedder: {
embed: (value) => {
inputs.push(value);
return inputs.length === 1 ? [1, 0] : [0, 1];
},
},
});
expect(inputs).toEqual(expected.map((sentence) => sentence.trim()));
expect(chunks.map((chunk) => chunk.content)).toEqual(inputs);
});

it("merges adjacent chunks when similarity threshold is met", async () => {
const text =
"Cats are great pets. Felines make wonderful companions. Quantum physics is unrelated to cats.";
Expand Down
7 changes: 7 additions & 0 deletions packages/rag/src/__tests__/sentence-chunker.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,13 @@ import { describe, expect, it } from "vitest";
import { SentenceChunker } from "../chunkers/sentence-chunker";

describe("SentenceChunker", () => {
it("keeps decimals intact with the default sentence tokenizer", () => {
const text = "Version 2.5 is out. It costs $3.99 today.";
const expected = ["Version 2.5 is out.", " It costs $3.99 today."];
const chunks = new SentenceChunker().chunk(text, { maxTokens: 4 });
expect(chunks.map((chunk) => chunk.content)).toEqual(expected);
});

it("groups sentences under token budget", () => {
const chunker = new SentenceChunker();
const text =
Expand Down
27 changes: 27 additions & 0 deletions packages/rag/src/__tests__/utils.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,33 @@ describe("Tokenizer utils", () => {
});

describe("Text utils", () => {
it.each([
[
"Version 2.5 is out. It costs $3.99 today.",
["Version 2.5 is out.", " It costs $3.99 today."],
],
["2.5", ["2.5"]],
["2.5.", ["2.5."]],
["Version 1.2.3 is ready! Next?", ["Version 1.2.3 is ready!", " Next?"]],
["A -2.5 change. Another 0.25 change.", ["A -2.5 change.", " Another 0.25 change."]],
[" Price $3.99.\nNext 2.50? ", [" Price $3.99.", "\nNext 2.50?"]],
["foo.bar baz.qux end.", ["foo.", "bar baz.", "qux end."]],
["One.Two!Three?", ["One.", "Two!", "Three?"]],
["1. 2.", ["1.", " 2."]],
["1...2!", ["1...", "2!"]],
["word.2 ends.", ["word.", "2 ends."]],
["No punctuation", ["No punctuation"]],
["", []],
[" ", []],
])("preserves sentence boundaries in %j", (text, expected) => {
const segments = splitIntoSentences(text);
expect(segments.map((segment) => segment.text)).toEqual(expected);
for (const segment of segments) {
expect(text.slice(segment.start, segment.end)).toBe(segment.text);
expect(segment.tokens).toBe(segment.text.trim().split(/\s+/).length);
}
});

it("splits into sentences with fallback for no punctuation", () => {
const sentences = splitIntoSentences("No punctuation sentence");
expect(sentences.length).toBe(1);
Expand Down
12 changes: 9 additions & 3 deletions packages/rag/src/utils/text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,16 @@ export function splitIntoSentences(
const segments: Segment[] = [];
const regex = new RegExp(SENTENCE_REGEX);
for (let match = regex.exec(text); match !== null; match = regex.exec(text)) {
const sentence = match[0]?.trim() ?? "";
if (!sentence) continue;
const start = match.index;
const end = match.index + (match[0]?.length ?? 0);
let end = match.index + (match[0]?.length ?? 0);
// A period between digits belongs to the same sentence (e.g. 2.5 or $3.99).
while (text[end - 1] === "." && /\d/.test(text[end - 2] ?? "") && /\d/.test(text[end] ?? "")) {
const continuation = regex.exec(text);
if (!continuation) break;
end = continuation.index + continuation[0].length;
}
const sentence = text.slice(start, end).trim();
if (!sentence) continue;
segments.push({
text: text.slice(start, end),
start,
Expand Down