Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/markdown-source-ranges.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@voltagent/rag": patch
---

fix(rag): map MarkdownChunker ranges and position metadata back to the original Markdown source, including sections, CRLF, list/quote markers, and fenced code. Addresses the Markdown offsets portion of #1542.
72 changes: 72 additions & 0 deletions packages/rag/src/__tests__/markdown-chunker.spec.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { describe, expect, it } from "vitest";
import { MarkdownChunker } from "../chunkers/markdown-chunker";
import { buildLineMap, positionForRange } from "../utils/position";
import { whitespaceTokenizer } from "../utils/tokenizer";

describe("MarkdownChunker", () => {
it("splits markdown by headings and chunks sections", () => {
Expand Down Expand Up @@ -36,3 +38,73 @@ console.log("hi");
).toBe(true);
});
});

describe("MarkdownChunker source offsets (#1542)", () => {
const chunker = new MarkdownChunker(whitespaceTokenizer);

it("maps separate sections to their original document ranges", () => {
const md = "# Intro\n\nFirst section.\n\n## Details\n\nSecond section.";
const chunks = chunker.chunk(md, { maxTokens: 50 });
expect(chunks.map(({ start, end }) => [start, end])).toEqual([
[9, 23],
[37, 52],
]);
for (const chunk of chunks) {
expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content);
expect(chunk.metadata?.position).toEqual(
positionForRange(chunk.start, chunk.end, buildLineMap(md)).position,
);
}
});

it("keeps raw CRLF, whitespace and Unicode offsets, including repeated content", () => {
const md =
" \r\n# Heading\r\n\r\n cafe\u0301 here. \r\n\r\n## Next\r\n\r\n cafe\u0301 here. ";
const chunks = chunker.chunk(md, { maxTokens: 50 });
expect(chunks).toHaveLength(2);
expect(chunks[0]?.start).toBe(md.indexOf("cafe"));
expect(chunks[1]?.start).toBe(md.lastIndexOf("cafe"));
for (const chunk of chunks) {
expect(md.slice(chunk.start, chunk.end)).toBe(chunk.content);
}
});

it("maps joined paragraph, list and quote content back to source spans", () => {
const md =
"# Heading\nfirst line\nsecond line\n\n- first item\n- second item\n\n> first quote\n> second quote";
const chunks = chunker.chunk(md, { maxTokens: 50 });
expect(chunks.map(({ content }) => content)).toEqual([
"first line second line",
"first item second item",
"first quote second quote",
]);
expect(chunks.map(({ start, end }) => md.slice(start, end))).toEqual([
"first line\nsecond line",
"first item\n- second item",
"first quote\n> second quote",
]);
});

it("maps token-split list items after removed markers, even when text repeats a marker", () => {
const md = "# Heading\n- - alpha beta\n- gamma delta epsilon";
const chunks = chunker.chunk(md, { maxTokens: 1 });
expect(chunks.length).toBeGreaterThan(1);
expect(chunks[0]?.start).toBe(md.indexOf("- alpha"));
for (const chunk of chunks) {
expect(md.slice(chunk.start, chunk.end).trim()).toBe(chunk.content.trim());
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
}
});

it("maps code chunks to the original fenced block and keeps indentation", () => {
const md = "# Code\r\n\r\n```ts\r\n const alpha = 1;\r\n const beta = 2;\r\n```";
for (const maxTokens of [50, 2]) {
const chunks = chunker.chunk(md, { maxTokens });
expect(chunks.length).toBeGreaterThan(0);
for (const chunk of chunks) {
expect(chunk.metadata?.blockType).toBe("code");
expect(md.slice(chunk.start, chunk.end).replace(/\r\n/g, "\n")).toBe(chunk.content);
expect(chunk.start).toBeGreaterThan(md.indexOf("```ts"));
}
}
});
});
78 changes: 58 additions & 20 deletions packages/rag/src/chunkers/markdown-chunker.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import type { Chunk, ChunkMetadataInput, Chunker, Tokenizer } from "../types";
import { buildMetadata } from "../utils/metadata";
import { normalizeText } from "../utils/text";
import { buildLineMap, positionForRange } from "../utils/position";
import { defaultTokenizer } from "../utils/tokenizer";
import { CodeChunker } from "./code-chunker";
import { RecursiveChunker } from "./recursive-chunker";
Expand All @@ -13,8 +13,8 @@ export type MarkdownChunkerOptions = {

type MdBlock =
| { type: "heading"; level: number; text: string }
| { type: "code"; language?: string; content: string }
| { type: "list" | "blockquote" | "paragraph"; content: string };
| { type: "code"; language?: string; content: string; offsets: number[] }
| { type: "list" | "blockquote" | "paragraph"; content: string; offsets: number[] };

type Section = {
headingPath: string[];
Expand All @@ -23,13 +23,40 @@ type Section = {

function parseBlocks(markdown: string): MdBlock[] {
const blocks: MdBlock[] = [];
const lines = markdown.split(/\r?\n/);
const lines = markdown.split(/\r\n|\r|\n/);
const lineStarts: number[] = [];
let lineStart = 0;
for (const line of lines) {
lineStarts.push(lineStart);
const newlineLength = markdown.startsWith("\r\n", lineStart + line.length) ? 2 : 1;
lineStart += line.length + newlineLength;
}
const mappedContent = (parts: { text: string; start: number }[], separator: string) => {
const content = parts.map((part) => part.text).join(separator);
// One original UTF-16 source offset per extracted content code unit.
const offsets: number[] = [];
parts.forEach((part, index) => {
if (index > 0) offsets.push(part.start - 1);
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
Resolved
for (let j = 0; j < part.text.length; j++) offsets.push(part.start + j);
});
return { content, offsets };
};
const trimmedLine = (index: number, prefix?: RegExp) => {
const raw = lines[index] ?? "";
const withoutPrefix = prefix ? raw.replace(prefix, "") : raw;
const text = withoutPrefix.trim();
return {
text,
start:
(lineStarts[index] ?? 0) + raw.length - withoutPrefix.length + withoutPrefix.indexOf(text),
};
};
let i = 0;
let buffer: string[] = [];
let buffer: { text: string; start: number }[] = [];
const flushParagraph = () => {
const text = buffer.join(" ").trim();
if (text) {
blocks.push({ type: "paragraph", content: text });
const mapped = mappedContent(buffer, " ");
if (mapped.content) {
blocks.push({ type: "paragraph", ...mapped });
}
buffer = [];
};
Expand All @@ -55,38 +82,38 @@ function parseBlocks(markdown: string): MdBlock[] {
if (codeFence) {
flushParagraph();
const language = codeFence[1]?.trim();
const contentLines: string[] = [];
const contentLines: { text: string; start: number }[] = [];
i += 1;
while (i < lines.length && !/^```/.test(lines[i] ?? "")) {
const current = lines[i];
if (current === undefined) break;
contentLines.push(current);
contentLines.push({ text: current, start: lineStarts[i] ?? 0 });
i += 1;
}
blocks.push({ type: "code", language, content: contentLines.join("\n") });
blocks.push({ type: "code", language, ...mappedContent(contentLines, "\n") });
i += 1;
continue;
}

if (/^>/.test(trimmed)) {
flushParagraph();
const quoteLines: string[] = [];
const quoteLines: { text: string; start: number }[] = [];
while (i < lines.length && /^>/.test(lines[i]?.trim() ?? "")) {
quoteLines.push((lines[i] ?? "").replace(/^>\s?/, ""));
quoteLines.push(trimmedLine(i, /^\s*>\s?/));
i += 1;
}
blocks.push({ type: "blockquote", content: quoteLines.join(" ").trim() });
blocks.push({ type: "blockquote", ...mappedContent(quoteLines, " ") });
continue;
}

if (/^(-|\*|\d+\.)\s+/.test(trimmed)) {
flushParagraph();
const listLines: string[] = [];
const listLines: { text: string; start: number }[] = [];
while (i < lines.length && /^(-|\*|\d+\.)\s+/.test(lines[i]?.trim() ?? "")) {
listLines.push((lines[i] ?? "").replace(/^(-|\*|\d+\.)\s+/, ""));
listLines.push(trimmedLine(i, /^\s*(-|\*|\d+\.)\s+/));
i += 1;
}
blocks.push({ type: "list", content: listLines.join(" ").trim() });
blocks.push({ type: "list", ...mappedContent(listLines, " ") });
continue;
}

Expand All @@ -96,7 +123,7 @@ function parseBlocks(markdown: string): MdBlock[] {
continue;
}

buffer.push(trimmed);
buffer.push(trimmedLine(i));
i += 1;
}
flushParagraph();
Expand Down Expand Up @@ -147,8 +174,10 @@ export class MarkdownChunker implements Chunker<MarkdownChunkerOptions> {
const label = options?.label ?? "markdown";
const baseMeta = options;

const normalized = normalizeText(markdown);
const sections = toSections(normalized);
// Parse the original source: normalizing the whole document first changes
// offsets (CRLF, indentation, leading whitespace, Unicode) before we can map them.
const sections = toSections(markdown);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
const lineMap = buildLineMap(markdown);
const chunks: Chunk[] = [];
let blockIndex = 0;

Expand Down Expand Up @@ -179,8 +208,16 @@ export class MarkdownChunker implements Chunker<MarkdownChunkerOptions> {
}

produced.forEach((chunk, idx) => {
// Chunk offsets refer to the extracted block, whose newlines/markers
// may have been removed. Map both boundaries back to the source.
// A span crossing lines may include source markers/newlines that
// are intentionally absent from the extracted chunk content.
const start = block.offsets[chunk.start] ?? 0;
const end = (block.offsets[chunk.end - 1] ?? start - 1) + 1;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
chunks.push({
...chunk,
start,
end,
id: `${label}-${index}-${bIdx}-${idx}`,
metadata: {
...chunk.metadata,
Expand All @@ -190,6 +227,7 @@ export class MarkdownChunker implements Chunker<MarkdownChunkerOptions> {
path: section.headingPath,
base: baseMeta,
extra: {
...positionForRange(start, end, lineMap),
headingPath: section.headingPath,
blockType: block.type,
blockIndex,
Expand Down