diff --git a/core/indexing/chunk/markdown.ts b/core/indexing/chunk/markdown.ts index 5a0df6776d8..b3b3765fdf9 100644 --- a/core/indexing/chunk/markdown.ts +++ b/core/indexing/chunk/markdown.ts @@ -133,7 +133,9 @@ export async function* markdownChunker( hLevel + 1, )) { yield { - content: `${section.header}\n${chunk.content}`, + content: section.header + ? `${section.header}\n${chunk.content}` + : chunk.content, startLine: section.startLine + chunk.startLine, endLine: section.startLine + chunk.endLine, otherMetadata: { diff --git a/core/indexing/chunk/markdown.vitest.ts b/core/indexing/chunk/markdown.vitest.ts new file mode 100644 index 00000000000..e7eb791b8f7 --- /dev/null +++ b/core/indexing/chunk/markdown.vitest.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("../../llm/countTokens", () => ({ + countTokens: (content: string) => content.length, + countTokensAsync: async (content: string) => content.length, +})); + +import { ChunkWithoutID } from "../../index"; + +import { markdownChunker } from "./markdown"; + +async function collectChunks( + content: string, + maxChunkSize: number, + hLevel: number, +): Promise { + const chunks: ChunkWithoutID[] = []; + for await (const chunk of markdownChunker(content, maxChunkSize, hLevel)) { + chunks.push(chunk); + } + return chunks; +} + +describe("markdownChunker", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should not prefix chunks with 'undefined' when a section has no header", async () => { + const content = Array.from( + { length: 10 }, + (_, i) => `Line ${i} of a markdown document that has no headers at all.`, + ).join("\n"); + + const chunks = await collectChunks(content, 100, 0); + + expect(chunks.length).toBeGreaterThan(0); + for (const chunk of chunks) { + expect(chunk.content).not.toContain("undefined"); + } + }); + + it("should not prefix headerless sub-sections with 'undefined'", async () => { + const content = [ + "## Installation", + ...Array.from( + { length: 10 }, + (_, i) => `Installation step ${i} with a bit of extra detail.`, + ), + ].join("\n"); + + const chunks = await collectChunks(content, 120, 1); + + expect(chunks.length).toBeGreaterThan(0); + for (const chunk of chunks) { + expect(chunk.content).not.toContain("undefined"); + expect(chunk.content.startsWith("## Installation\n")).toBe(true); + } + }); + + it("should still prepend the section header to each chunk", async () => { + const content = [ + "## Installation", + ...Array.from( + { length: 10 }, + (_, i) => `Installation step ${i} with a bit of extra detail.`, + ), + ].join("\n"); + + const chunks = await collectChunks(content, 120, 1); + + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) { + expect(chunk.otherMetadata?.title).toBe("Installation"); + } + }); +});