Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/preserve-decimal-sentences.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/rag": patch
---

Preserve decimal numbers and prices when splitting sentences. Refs #1542.
39 changes: 39 additions & 0 deletions packages/rag/src/__tests__/decimal-sentences.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { RecursiveChunker } from "../chunkers/recursive-chunker";
import { SemanticChunker } from "../chunkers/semantic-chunker";
import { SentenceChunker } from "../chunkers/sentence-chunker";
import { whitespaceTokenizer } from "../utils/tokenizer";

const text = "Version 2.5 is out. It costs $3.99 today.";
const expected = ["Version 2.5 is out.", " It costs $3.99 today."];

describe("decimal sentence boundaries", () => {
it("keeps decimals intact with the default sentence tokenizer", () => {
const chunks = new SentenceChunker().chunk(text, { maxTokens: 4 });
expect(chunks.map((chunk) => chunk.content)).toEqual(expected);
});

it("keeps decimals intact when recursively splitting a paragraph", () => {
const chunks = new RecursiveChunker(whitespaceTokenizer).chunk(text, { maxTokens: 4 });
expect(chunks.map((chunk) => chunk.content)).toEqual([
expected[0],
expected[0],
expected[1]?.trim(),
]);
});

it("passes complete decimal sentences to the semantic embedder", async () => {
const inputs: string[] = [];
const chunks = await new SemanticChunker(whitespaceTokenizer).chunk(text, {
maxTokens: 4,
embedder: {
embed: (value) => {
inputs.push(value);
return inputs.length === 1 ? [1, 0] : [0, 1];
},
},
});
expect(inputs).toEqual(expected.map((sentence) => sentence.trim()));
expect(chunks.map((chunk) => chunk.content)).toEqual(inputs);
});
});
27 changes: 27 additions & 0 deletions packages/rag/src/__tests__/utils.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,33 @@ describe("Tokenizer utils", () => {
});

describe("Text utils", () => {
it.each([
[
"Version 2.5 is out. It costs $3.99 today.",
["Version 2.5 is out.", " It costs $3.99 today."],
],
["2.5", ["2.5"]],
["2.5.", ["2.5."]],
["Version 1.2.3 is ready! Next?", ["Version 1.2.3 is ready!", " Next?"]],
["A -2.5 change. Another 0.25 change.", ["A -2.5 change.", " Another 0.25 change."]],
[" Price $3.99.\nNext 2.50? ", [" Price $3.99.", "\nNext 2.50?"]],
["foo.bar baz.qux end.", ["foo.", "bar baz.", "qux end."]],
["One.Two!Three?", ["One.", "Two!", "Three?"]],
["1. 2.", ["1.", " 2."]],
["1...2!", ["1...", "2!"]],
["word.2 ends.", ["word.", "2 ends."]],
["No punctuation", ["No punctuation"]],
["", []],
[" ", []],
])("preserves sentence boundaries in %j", (text, expected) => {
const segments = splitIntoSentences(text);
expect(segments.map((segment) => segment.text)).toEqual(expected);
for (const segment of segments) {
expect(text.slice(segment.start, segment.end)).toBe(segment.text);
expect(segment.tokens).toBe(segment.text.trim().split(/\s+/).length);
}
});

it("splits into sentences with fallback for no punctuation", () => {
const sentences = splitIntoSentences("No punctuation sentence");
expect(sentences.length).toBe(1);
Expand Down
12 changes: 9 additions & 3 deletions packages/rag/src/utils/text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,16 @@ export function splitIntoSentences(
const segments: Segment[] = [];
const regex = new RegExp(SENTENCE_REGEX);
for (let match = regex.exec(text); match !== null; match = regex.exec(text)) {
const sentence = match[0]?.trim() ?? "";
if (!sentence) continue;
const start = match.index;
const end = match.index + (match[0]?.length ?? 0);
let end = match.index + (match[0]?.length ?? 0);
// A period between digits belongs to the same sentence (e.g. 2.5 or $3.99).
while (text[end - 1] === "." && /\d/.test(text[end - 2] ?? "") && /\d/.test(text[end] ?? "")) {
const continuation = regex.exec(text);
if (!continuation) break;
end = continuation.index + continuation[0].length;
}
const sentence = text.slice(start, end).trim();
if (!sentence) continue;
segments.push({
text: text.slice(start, end),
start,
Expand Down