diff --git a/.semversioner/next-release/patch-20260710221345787890.json b/.semversioner/next-release/patch-20260710221345787890.json new file mode 100644 index 000000000..dc8fc4c00 --- /dev/null +++ b/.semversioner/next-release/patch-20260710221345787890.json @@ -0,0 +1,4 @@ +{ + "type": "patch", + "description": "Fix JSONL input loader to skip blank lines and ignore invalid JSON rows" +} diff --git a/packages/graphrag-input/graphrag_input/jsonl.py b/packages/graphrag-input/graphrag_input/jsonl.py index f038aafaa..053b6deaf 100644 --- a/packages/graphrag-input/graphrag_input/jsonl.py +++ b/packages/graphrag-input/graphrag_input/jsonl.py @@ -34,5 +34,28 @@ async def read_file(self, path: str) -> list[TextDocument]: - output - list with a TextDocument for each row in the file. """ text = await self._storage.get(path, encoding=self._encoding) - rows = [json.loads(line) for line in text.splitlines()] + rows: list[dict] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + + try: + parsed_row = json.loads(line) + except json.JSONDecodeError: + logger.warning( + "Skipping malformed JSONL row in %s at line %s", + path, + line_number, + ) + continue + + if isinstance(parsed_row, dict): + rows.append(parsed_row) + else: + logger.warning( + "Skipping non-object JSONL row in %s at line %s", + path, + line_number, + ) + return await self.process_data_columns(rows, path) diff --git a/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl b/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl new file mode 100644 index 000000000..a8f5f494b --- /dev/null +++ b/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl @@ -0,0 +1,7 @@ +{ "title": "Hello", "text": "Hi how are you today?"} + +not json +{ "title": "Goodbye", "text": "I'm outta here"} + +["not", "an", "object"] +{ "title": "Adios", "text": "See you later"} diff --git a/tests/unit/indexing/input/test_jsonl_loader.py b/tests/unit/indexing/input/test_jsonl_loader.py index dd5609429..bad26f046 100644 --- a/tests/unit/indexing/input/test_jsonl_loader.py +++ b/tests/unit/indexing/input/test_jsonl_loader.py @@ -40,3 +40,20 @@ async def test_jsonl_loader_one_file_with_title(): documents = await reader.read_files() assert len(documents) == 3 assert documents[0].title == "Hello" + + +async def test_jsonl_loader_skips_blank_and_invalid_rows(): + config = InputConfig( + type=InputType.JsonLines, + title_column="title", + ) + storage = create_storage( + StorageConfig( + base_dir="tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines", + ) + ) + reader = create_input_reader(config, storage) + documents = await reader.read_files() + + assert len(documents) == 3 + assert [document.title for document in documents] == ["Hello", "Goodbye", "Adios"]