diff --git a/.changeset/unique-code-ast-chunk-ids.md b/.changeset/unique-code-ast-chunk-ids.md new file mode 100644 index 000000000..60c6f4b8c --- /dev/null +++ b/.changeset/unique-code-ast-chunk-ids.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +Keep code chunk IDs unique when token splitting expands an AST block before other blocks. diff --git a/packages/rag/src/chunkers/code-chunker-ast-ids.spec.ts b/packages/rag/src/chunkers/code-chunker-ast-ids.spec.ts new file mode 100644 index 000000000..9d47489f8 --- /dev/null +++ b/packages/rag/src/chunkers/code-chunker-ast-ids.spec.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from "vitest"; +import { whitespaceTokenizer } from "../utils/tokenizer"; +import { CodeChunker } from "./code-chunker"; + +describe("CodeChunker AST expansion", () => { + it("assigns unique IDs when a split AST block precedes smaller blocks", () => { + const blocks = ["one two three four five", "six", "seven"]; + const text = ["```txt", ...blocks, "```"].join("\n"); + const chunks = new CodeChunker(whitespaceTokenizer).chunk(text, { + maxTokens: 2, + parser: (source) => + blocks.map((block) => { + const start = source.indexOf(block); + return { kind: "function", start, end: start + block.length }; + }), + }); + + expect(chunks.map((chunk) => chunk.content)).toEqual([ + "one two", + "three four", + "five", + "six", + "seven", + ]); + expect(chunks.map((chunk) => chunk.id)).toEqual([ + "code-0", + "code-1", + "code-2", + "code-3", + "code-4", + ]); + }); +}); diff --git a/packages/rag/src/chunkers/code-chunker.ts b/packages/rag/src/chunkers/code-chunker.ts index 838a2c333..6b268a298 100644 --- a/packages/rag/src/chunkers/code-chunker.ts +++ b/packages/rag/src/chunkers/code-chunker.ts @@ -119,10 +119,10 @@ export class CodeChunker implements Chunker { tokenizer, label: `${label}-ast`, }); - tokenChunks.forEach((chunk, idx) => { + tokenChunks.forEach((chunk) => { chunks.push({ ...chunk, - id: `${label}-${chunks.length + idx}`, + id: `${label}-${chunks.length}`, start: segment.start + block.start + chunk.start, end: segment.start + block.start + chunk.end, metadata: buildMetadata({