From 2ef17d52af48f72be9ec1dc89dc93f58476d552a Mon Sep 17 00:00:00 2001 From: tomwangcn Date: Sat, 14 Nov 2026 15:19:25 +0000 Subject: [PATCH 1/3] fix(rag): map Markdown chunks to original source ranges --- .changeset/markdown-source-ranges.md | 5 ++ .../src/__tests__/markdown-chunker.spec.ts | 72 +++++++++++++++++ packages/rag/src/chunkers/markdown-chunker.ts | 78 ++++++++++++++----- 3 files changed, 135 insertions(+), 20 deletions(-) create mode 100644 .changeset/markdown-source-ranges.md diff --git a/.changeset/markdown-source-ranges.md b/.changeset/markdown-source-ranges.md new file mode 100644 index 000000000..bb8322d92 --- /dev/null +++ b/.changeset/markdown-source-ranges.md @@ -0,0 +1,5 @@ +--- +"@voltagent/rag": patch +--- + +fix(rag): map MarkdownChunker ranges and position metadata back to the original Markdown source, including sections, CRLF, list/quote markers, and fenced code. Addresses the Markdown offsets portion of #1542. diff --git a/packages/rag/src/__tests__/markdown-chunker.spec.ts b/packages/rag/src/__tests__/markdown-chunker.spec.ts index c253bd5ce..613a1ac3d 100644 --- a/packages/rag/src/__tests__/markdown-chunker.spec.ts +++ b/packages/rag/src/__tests__/markdown-chunker.spec.ts @@ -1,5 +1,7 @@ import { describe, expect, it } from "vitest"; import { MarkdownChunker } from "../chunkers/markdown-chunker"; +import { buildLineMap, positionForRange } from "../utils/position"; +import { whitespaceTokenizer } from "../utils/tokenizer"; describe("MarkdownChunker", () => { it("splits markdown by headings and chunks sections", () => { @@ -36,3 +38,73 @@ console.log("hi"); ).toBe(true); }); }); + +describe("MarkdownChunker source offsets (#1542)", () => { + const chunker = new MarkdownChunker(whitespaceTokenizer); + + it("maps separate sections to their original document ranges", () => { + const md = "# Intro\n\nFirst section.\n\n## Details\n\nSecond section."; + const chunks = chunker.chunk(md, { maxTokens: 50 }); + expect(chunks.map(({ start, end }) => [start, end])).toEqual([ + [9, 23], + [37, 52], + ]); + for (const chunk of chunks) { + expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content); + expect(chunk.metadata?.position).toEqual( + positionForRange(chunk.start, chunk.end, buildLineMap(md)).position, + ); + } + }); + + it("keeps raw CRLF, whitespace and Unicode offsets, including repeated content", () => { + const md = + " \r\n# Heading\r\n\r\n cafe\u0301 here. \r\n\r\n## Next\r\n\r\n cafe\u0301 here. "; + const chunks = chunker.chunk(md, { maxTokens: 50 }); + expect(chunks).toHaveLength(2); + expect(chunks[0]?.start).toBe(md.indexOf("cafe")); + expect(chunks[1]?.start).toBe(md.lastIndexOf("cafe")); + for (const chunk of chunks) { + expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content); + } + }); + + it("maps joined paragraph, list and quote content back to source spans", () => { + const md = + "# Heading\nfirst line\nsecond line\n\n- first item\n- second item\n\n> first quote\n> second quote"; + const chunks = chunker.chunk(md, { maxTokens: 50 }); + expect(chunks.map(({ content }) => content)).toEqual([ + "first line second line", + "first item second item", + "first quote second quote", + ]); + expect(chunks.map(({ start, end }) => md.slice(start, end))).toEqual([ + "first line\nsecond line", + "first item\n- second item", + "first quote\n> second quote", + ]); + }); + + it("maps token-split list items after removed markers, even when text repeats a marker", () => { + const md = "# Heading\n- - alpha beta\n- gamma delta epsilon"; + const chunks = chunker.chunk(md, { maxTokens: 1 }); + expect(chunks.length).toBeGreaterThan(1); + expect(chunks[0]?.start).toBe(md.indexOf("- alpha")); + for (const chunk of chunks) { + expect(md.slice(chunk.start, chunk.end).trim()).toBe(chunk.content.trim()); + } + }); + + it("maps code chunks to the original fenced block and keeps indentation", () => { + const md = "# Code\r\n\r\n```ts\r\n const alpha = 1;\r\n const beta = 2;\r\n```"; + for (const maxTokens of [50, 2]) { + const chunks = chunker.chunk(md, { maxTokens }); + expect(chunks.length).toBeGreaterThan(0); + for (const chunk of chunks) { + expect(chunk.metadata?.blockType).toBe("code"); + expect(md.slice(chunk.start, chunk.end).replace(/\r\n/g, "\n")).toBe(chunk.content); + expect(chunk.start).toBeGreaterThan(md.indexOf("```ts")); + } + } + }); +}); diff --git a/packages/rag/src/chunkers/markdown-chunker.ts b/packages/rag/src/chunkers/markdown-chunker.ts index 2eaedc1d9..a3950207d 100644 --- a/packages/rag/src/chunkers/markdown-chunker.ts +++ b/packages/rag/src/chunkers/markdown-chunker.ts @@ -1,6 +1,6 @@ import type { Chunk, ChunkMetadataInput, Chunker, Tokenizer } from "../types"; import { buildMetadata } from "../utils/metadata"; -import { normalizeText } from "../utils/text"; +import { buildLineMap, positionForRange } from "../utils/position"; import { defaultTokenizer } from "../utils/tokenizer"; import { CodeChunker } from "./code-chunker"; import { RecursiveChunker } from "./recursive-chunker"; @@ -13,8 +13,8 @@ export type MarkdownChunkerOptions = { type MdBlock = | { type: "heading"; level: number; text: string } - | { type: "code"; language?: string; content: string } - | { type: "list" | "blockquote" | "paragraph"; content: string }; + | { type: "code"; language?: string; content: string; offsets: number[] } + | { type: "list" | "blockquote" | "paragraph"; content: string; offsets: number[] }; type Section = { headingPath: string[]; @@ -23,13 +23,40 @@ type Section = { function parseBlocks(markdown: string): MdBlock[] { const blocks: MdBlock[] = []; - const lines = markdown.split(/\r?\n/); + const lines = markdown.split(/\r\n|\r|\n/); + const lineStarts: number[] = []; + let lineStart = 0; + for (const line of lines) { + lineStarts.push(lineStart); + const newlineLength = markdown.startsWith("\r\n", lineStart + line.length) ? 2 : 1; + lineStart += line.length + newlineLength; + } + const mappedContent = (parts: { text: string; start: number }[], separator: string) => { + const content = parts.map((part) => part.text).join(separator); + // One original UTF-16 source offset per extracted content code unit. + const offsets: number[] = []; + parts.forEach((part, index) => { + if (index > 0) offsets.push(part.start - 1); + for (let j = 0; j < part.text.length; j++) offsets.push(part.start + j); + }); + return { content, offsets }; + }; + const trimmedLine = (index: number, prefix?: RegExp) => { + const raw = lines[index] ?? ""; + const withoutPrefix = prefix ? raw.replace(prefix, "") : raw; + const text = withoutPrefix.trim(); + return { + text, + start: + (lineStarts[index] ?? 0) + raw.length - withoutPrefix.length + withoutPrefix.indexOf(text), + }; + }; let i = 0; - let buffer: string[] = []; + let buffer: { text: string; start: number }[] = []; const flushParagraph = () => { - const text = buffer.join(" ").trim(); - if (text) { - blocks.push({ type: "paragraph", content: text }); + const mapped = mappedContent(buffer, " "); + if (mapped.content) { + blocks.push({ type: "paragraph", ...mapped }); } buffer = []; }; @@ -55,38 +82,38 @@ function parseBlocks(markdown: string): MdBlock[] { if (codeFence) { flushParagraph(); const language = codeFence[1]?.trim(); - const contentLines: string[] = []; + const contentLines: { text: string; start: number }[] = []; i += 1; while (i < lines.length && !/^```/.test(lines[i] ?? "")) { const current = lines[i]; if (current === undefined) break; - contentLines.push(current); + contentLines.push({ text: current, start: lineStarts[i] ?? 0 }); i += 1; } - blocks.push({ type: "code", language, content: contentLines.join("\n") }); + blocks.push({ type: "code", language, ...mappedContent(contentLines, "\n") }); i += 1; continue; } if (/^>/.test(trimmed)) { flushParagraph(); - const quoteLines: string[] = []; + const quoteLines: { text: string; start: number }[] = []; while (i < lines.length && /^>/.test(lines[i]?.trim() ?? "")) { - quoteLines.push((lines[i] ?? "").replace(/^>\s?/, "")); + quoteLines.push(trimmedLine(i, /^\s*>\s?/)); i += 1; } - blocks.push({ type: "blockquote", content: quoteLines.join(" ").trim() }); + blocks.push({ type: "blockquote", ...mappedContent(quoteLines, " ") }); continue; } if (/^(-|\*|\d+\.)\s+/.test(trimmed)) { flushParagraph(); - const listLines: string[] = []; + const listLines: { text: string; start: number }[] = []; while (i < lines.length && /^(-|\*|\d+\.)\s+/.test(lines[i]?.trim() ?? "")) { - listLines.push((lines[i] ?? "").replace(/^(-|\*|\d+\.)\s+/, "")); + listLines.push(trimmedLine(i, /^\s*(-|\*|\d+\.)\s+/)); i += 1; } - blocks.push({ type: "list", content: listLines.join(" ").trim() }); + blocks.push({ type: "list", ...mappedContent(listLines, " ") }); continue; } @@ -96,7 +123,7 @@ function parseBlocks(markdown: string): MdBlock[] { continue; } - buffer.push(trimmed); + buffer.push(trimmedLine(i)); i += 1; } flushParagraph(); @@ -147,8 +174,10 @@ export class MarkdownChunker implements Chunker { const label = options?.label ?? "markdown"; const baseMeta = options; - const normalized = normalizeText(markdown); - const sections = toSections(normalized); + // Parse the original source: normalizing the whole document first changes + // offsets (CRLF, indentation, leading whitespace, Unicode) before we can map them. + const sections = toSections(markdown); + const lineMap = buildLineMap(markdown); const chunks: Chunk[] = []; let blockIndex = 0; @@ -179,8 +208,16 @@ export class MarkdownChunker implements Chunker { } produced.forEach((chunk, idx) => { + // Chunk offsets refer to the extracted block, whose newlines/markers + // may have been removed. Map both boundaries back to the source. + // A span crossing lines may include source markers/newlines that + // are intentionally absent from the extracted chunk content. + const start = block.offsets[chunk.start] ?? 0; + const end = (block.offsets[chunk.end - 1] ?? start - 1) + 1; chunks.push({ ...chunk, + start, + end, id: `${label}-${index}-${bIdx}-${idx}`, metadata: { ...chunk.metadata, @@ -190,6 +227,7 @@ export class MarkdownChunker implements Chunker { path: section.headingPath, base: baseMeta, extra: { + ...positionForRange(start, end, lineMap), headingPath: section.headingPath, blockType: block.type, blockIndex, From f6a33ea9f1dd172ae567bf45d30680459ed79980 Mon Sep 17 00:00:00 2001 From: tomwangcn Date: Sat, 14 Nov 2026 15:27:44 +0000 Subject: [PATCH 2/3] fix(rag): retain fence boundaries and sentence-group source origins --- .../rag/src/__tests__/markdown-chunker.spec.ts | 18 +++++++++++++++++- packages/rag/src/chunkers/markdown-chunker.ts | 9 +++++++-- packages/rag/src/chunkers/recursive-chunker.ts | 8 ++++++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/packages/rag/src/__tests__/markdown-chunker.spec.ts b/packages/rag/src/__tests__/markdown-chunker.spec.ts index 613a1ac3d..587346395 100644 --- a/packages/rag/src/__tests__/markdown-chunker.spec.ts +++ b/packages/rag/src/__tests__/markdown-chunker.spec.ts @@ -91,7 +91,7 @@ describe("MarkdownChunker source offsets (#1542)", () => { expect(chunks.length).toBeGreaterThan(1); expect(chunks[0]?.start).toBe(md.indexOf("- alpha")); for (const chunk of chunks) { - expect(md.slice(chunk.start, chunk.end).trim()).toBe(chunk.content.trim()); + expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content); } }); @@ -108,3 +108,19 @@ describe("MarkdownChunker source offsets (#1542)", () => { } }); }); + +it("keeps indented closing fences from swallowing later sections", () => { + const md = "# Code\n ```ts\n const x = 1;\n ```\n\n## After\nVisible paragraph."; + const chunks = new MarkdownChunker(whitespaceTokenizer).chunk(md); + expect(chunks.at(-1)?.content).toBe("Visible paragraph."); + expect(chunks.at(-1)?.start).toBe(md.indexOf("Visible")); + expect(chunks.at(-1)?.metadata?.headingPath).toContain("After"); +}); + +it("maps later token-split sentence groups, not the first group again", () => { + const md = "# H\nA. B. C."; + const chunks = new MarkdownChunker(whitespaceTokenizer).chunk(md, { maxTokens: 1 }); + expect(chunks.at(-1)?.content).toBe("C."); + expect(chunks.at(-1)?.start).toBe(md.indexOf("C.")); + for (const chunk of chunks) expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content); +}); diff --git a/packages/rag/src/chunkers/markdown-chunker.ts b/packages/rag/src/chunkers/markdown-chunker.ts index a3950207d..9761b1888 100644 --- a/packages/rag/src/chunkers/markdown-chunker.ts +++ b/packages/rag/src/chunkers/markdown-chunker.ts @@ -36,7 +36,12 @@ function parseBlocks(markdown: string): MdBlock[] { // One original UTF-16 source offset per extracted content code unit. const offsets: number[] = []; parts.forEach((part, index) => { - if (index > 0) offsets.push(part.start - 1); + if (index > 0) { + const previous = parts[index - 1]; + // A joined separator represents the gap after the previous line, + // not the final character of the next line's stripped prefix. + offsets.push((previous?.start ?? 0) + (previous?.text.length ?? 0)); + } for (let j = 0; j < part.text.length; j++) offsets.push(part.start + j); }); return { content, offsets }; @@ -84,7 +89,7 @@ function parseBlocks(markdown: string): MdBlock[] { const language = codeFence[1]?.trim(); const contentLines: { text: string; start: number }[] = []; i += 1; - while (i < lines.length && !/^```/.test(lines[i] ?? "")) { + while (i < lines.length && !/^```/.test(lines[i]?.trim() ?? "")) { const current = lines[i]; if (current === undefined) break; contentLines.push({ text: current, start: lineStarts[i] ?? 0 }); diff --git a/packages/rag/src/chunkers/recursive-chunker.ts b/packages/rag/src/chunkers/recursive-chunker.ts index bbe8171b7..8af2c1681 100644 --- a/packages/rag/src/chunkers/recursive-chunker.ts +++ b/packages/rag/src/chunkers/recursive-chunker.ts @@ -87,6 +87,10 @@ export class RecursiveChunker implements Chunker { ]; } + // The sentence chunk's content may omit leading whitespace. Token + // offsets are relative to that content, not to the whole paragraph. + const groupStart = paragraph.text.indexOf(chunk.content, chunk.start); + const contentStart = groupStart >= 0 ? groupStart : chunk.start; // Fallback to token chunking for oversized sentence group return this.tokenChunker .chunk(chunk.content, { @@ -101,8 +105,8 @@ export class RecursiveChunker implements Chunker { .map((tokenChunk, idx) => ({ ...tokenChunk, id: `recursive-${chunks.length + idx}`, - start: paragraph.start + (tokenChunk.start ?? 0), - end: paragraph.start + (tokenChunk.end ?? 0), + start: paragraph.start + contentStart + (tokenChunk.start ?? 0), + end: paragraph.start + contentStart + (tokenChunk.end ?? 0), metadata: { ...tokenChunk.metadata, ...buildMetadata({ From d9a11cb7cb70ec63850ab0e9013cdde552ab9bc6 Mon Sep 17 00:00:00 2001 From: tomwangcn Date: Sat, 14 Nov 2026 21:18:10 +0000 Subject: [PATCH 3/3] fix(rag): recognize only complete raw closing fences --- packages/rag/src/__tests__/markdown-chunker.spec.ts | 12 ++++++++++++ packages/rag/src/chunkers/markdown-chunker.ts | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/rag/src/__tests__/markdown-chunker.spec.ts b/packages/rag/src/__tests__/markdown-chunker.spec.ts index 587346395..4dbdd0709 100644 --- a/packages/rag/src/__tests__/markdown-chunker.spec.ts +++ b/packages/rag/src/__tests__/markdown-chunker.spec.ts @@ -124,3 +124,15 @@ it("maps later token-split sentence groups, not the first group again", () => { expect(chunks.at(-1)?.start).toBe(md.indexOf("C.")); for (const chunk of chunks) expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content); }); + +it("does not discard a fence-like code line with an info suffix or excessive indentation", () => { + const md = + "# Code\n```ts\n ```literal\n# not a heading\n ```\nconst x = 1;\n```\n\n## After\nVisible."; + const chunks = new MarkdownChunker(whitespaceTokenizer).chunk(md, { maxTokens: 50 }); + expect(chunks.at(-1)?.metadata?.headingPath).toEqual(["Code", "After"]); + const code = chunks.filter((chunk) => chunk.metadata?.blockType === "code"); + expect(code.map((chunk) => chunk.content).join("\n")).toContain("# not a heading"); + expect( + chunks.filter((chunk) => chunk.metadata?.blockType !== "code").map((chunk) => chunk.content), + ).toEqual(["Visible."]); +}); diff --git a/packages/rag/src/chunkers/markdown-chunker.ts b/packages/rag/src/chunkers/markdown-chunker.ts index 9761b1888..13fea70f8 100644 --- a/packages/rag/src/chunkers/markdown-chunker.ts +++ b/packages/rag/src/chunkers/markdown-chunker.ts @@ -89,7 +89,7 @@ function parseBlocks(markdown: string): MdBlock[] { const language = codeFence[1]?.trim(); const contentLines: { text: string; start: number }[] = []; i += 1; - while (i < lines.length && !/^```/.test(lines[i]?.trim() ?? "")) { + while (i < lines.length && !/^ {0,3}`{3,}[\t ]*$/.test(lines[i] ?? "")) { const current = lines[i]; if (current === undefined) break; contentLines.push({ text: current, start: lineStarts[i] ?? 0 });