From 96dae42f6d75228f92e10945f52b5bc0a1c502af Mon Sep 17 00:00:00 2001 From: Nathan Evans Date: Fri, 10 Jul 2026 15:19:10 -0700 Subject: [PATCH 1/3] Fix JSONL loader for blank and invalid rows --- .../patch-20260710221345787890.json | 4 +++ .../graphrag-input/graphrag_input/jsonl.py | 25 ++++++++++++++++++- .../input.jsonl | 7 ++++++ .../unit/indexing/input/test_jsonl_loader.py | 17 +++++++++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .semversioner/next-release/patch-20260710221345787890.json create mode 100644 tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl diff --git a/.semversioner/next-release/patch-20260710221345787890.json b/.semversioner/next-release/patch-20260710221345787890.json new file mode 100644 index 0000000000..dc8fc4c00e --- /dev/null +++ b/.semversioner/next-release/patch-20260710221345787890.json @@ -0,0 +1,4 @@ +{ + "type": "patch", + "description": "Fix JSONL input loader to skip blank lines and ignore invalid JSON rows" +} diff --git a/packages/graphrag-input/graphrag_input/jsonl.py b/packages/graphrag-input/graphrag_input/jsonl.py index f038aafaa5..053b6deaf7 100644 --- a/packages/graphrag-input/graphrag_input/jsonl.py +++ b/packages/graphrag-input/graphrag_input/jsonl.py @@ -34,5 +34,28 @@ async def read_file(self, path: str) -> list[TextDocument]: - output - list with a TextDocument for each row in the file. """ text = await self._storage.get(path, encoding=self._encoding) - rows = [json.loads(line) for line in text.splitlines()] + rows: list[dict] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + continue + + try: + parsed_row = json.loads(line) + except json.JSONDecodeError: + logger.warning( + "Skipping malformed JSONL row in %s at line %s", + path, + line_number, + ) + continue + + if isinstance(parsed_row, dict): + rows.append(parsed_row) + else: + logger.warning( + "Skipping non-object JSONL row in %s at line %s", + path, + line_number, + ) + return await self.process_data_columns(rows, path) diff --git a/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl b/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl new file mode 100644 index 0000000000..a8f5f494b7 --- /dev/null +++ b/tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines/input.jsonl @@ -0,0 +1,7 @@ +{ "title": "Hello", "text": "Hi how are you today?"} + +not json +{ "title": "Goodbye", "text": "I'm outta here"} + +["not", "an", "object"] +{ "title": "Adios", "text": "See you later"} diff --git a/tests/unit/indexing/input/test_jsonl_loader.py b/tests/unit/indexing/input/test_jsonl_loader.py index dd56094290..bad26f0466 100644 --- a/tests/unit/indexing/input/test_jsonl_loader.py +++ b/tests/unit/indexing/input/test_jsonl_loader.py @@ -40,3 +40,20 @@ async def test_jsonl_loader_one_file_with_title(): documents = await reader.read_files() assert len(documents) == 3 assert documents[0].title == "Hello" + + +async def test_jsonl_loader_skips_blank_and_invalid_rows(): + config = InputConfig( + type=InputType.JsonLines, + title_column="title", + ) + storage = create_storage( + StorageConfig( + base_dir="tests/unit/indexing/input/data/jsonl-with-invalid-and-blank-lines", + ) + ) + reader = create_input_reader(config, storage) + documents = await reader.read_files() + + assert len(documents) == 3 + assert [document.title for document in documents] == ["Hello", "Goodbye", "Adios"] From 55f170f7d25f58ba4494f2dee8c5690fd282c778 Mon Sep 17 00:00:00 2001 From: Nathan Evans Date: Fri, 10 Jul 2026 15:57:55 -0700 Subject: [PATCH 2/3] Sync docs for schema and metadata fields --- docs/config/yaml.md | 14 +++++-------- docs/index/architecture.md | 2 +- docs/index/default_dataflow.md | 2 +- docs/index/inputs.md | 36 ++++++++++++++++++++++------------ docs/index/outputs.md | 7 ++++--- 5 files changed, 35 insertions(+), 26 deletions(-) diff --git a/docs/config/yaml.md b/docs/config/yaml.md index c3cf870097..7fa2a98657 100644 --- a/docs/config/yaml.md +++ b/docs/config/yaml.md @@ -74,7 +74,7 @@ embedding_models: ### input -Our pipeline can ingest .csv, .txt, or .json data from an input location. See the [inputs page](../index/inputs.md) for more details and examples. +Our pipeline can ingest `.csv`, `.txt`, `.json`, `.jsonl`, `.parquet`, or MarkItDown-supported files from an input location. See the [inputs page](../index/inputs.md) for more details and examples. #### Fields @@ -86,16 +86,16 @@ Our pipeline can ingest .csv, .txt, or .json data from an input location. See th - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. -- `type` **text|csv|json** - The type of input data to load. Default is `text` +- `type` **text|csv|json|jsonl|parquet|markitdown** - The type of input data to load. Default is `text` - `encoding` **str** - The encoding of the input file. Default is `utf-8` -- `file_pattern` **str** - A regex to match input files. Default is `.*\.csv$`, `.*\.txt$`, or `.*\.json$` depending on the specified `type`, but you can customize it if needed. +- `file_pattern` **str** - A regex to match input files. Defaults are derived from `type` (for example `.*\.csv$`, `.*\.txt$`, `.*\.json$`, `.*\.jsonl$`, `.*\.parquet$`), but you can customize it if needed. - `id_column` **str** - The input ID column to use. - `title_column` **str** - The input title column to use. - `text_column` **str** - The input text column to use. ### chunking -These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. Also note the `metadata` setting in the input document config, which will replicate document metadata into each chunk. +These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. You can also prepend selected document fields (standard fields like `title` or values from `raw_data`) into each chunk. #### Fields @@ -103,7 +103,7 @@ These settings configure how we parse documents into text chunks. This is necess - `encoding_model` **str** - The text encoding model to use for splitting on token boundaries. - `size` **int** - The max chunk size in tokens. - `overlap` **int** - The chunk overlap in tokens. -- `prepend_metadata` **list[str]** - Metadata fields from the source document to prepend on each chunk. +- `prepend_metadata` **list[str]** - Document fields to prepend on each chunk. These can be standard document fields (`id`, `title`, `text`, `creation_date`) or fields from the source row/object stored in `raw_data`. ## Outputs and Storage @@ -120,8 +120,6 @@ This section controls the storage mechanism used by the pipeline used for export - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. -- `type` **text|csv|json** - The type of input data to load. Default is `text` -- `encoding` **str** - The encoding of the input file. Default is `utf-8` ### update_output_storage @@ -136,8 +134,6 @@ The section defines a secondary storage location for running incremental indexin - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. -- `type` **text|csv|json** - The type of input data to load. Default is `text` -- `encoding` **str** - The encoding of the input file. Default is `utf-8` ### cache diff --git a/docs/index/architecture.md b/docs/index/architecture.md index c1c9e05f98..c5d86938fe 100644 --- a/docs/index/architecture.md +++ b/docs/index/architecture.md @@ -41,7 +41,7 @@ Several subsystems within GraphRAG use a factory pattern to register and retriev The following subsystems use a factory pattern that allows you to register your own implementations: - [language model](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-llm/graphrag_llm/completion/completion_factory.py) - implement your own `chat` and `embed` methods to use a model provider of choice beyond the built-in LiteLLM wrapper -- [input reader](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-input/graphrag_input/input_reader.py) - implement your own input document reader to support file types other than text, CSV, and JSON +- [input reader](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-input/graphrag_input/input_reader.py) - implement your own input document reader to support file types other than the built-in text, CSV, JSON, JSONL, Parquet, and MarkItDown readers - [cache](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-cache/graphrag_cache/cache_factory.py) - create your own cache storage location in addition to the file, blob, and CosmosDB ones we provide - [logger](https://github.com/microsoft/graphrag/blob/main/packages/graphrag/graphrag/logger/factory.py) - create your own log writing location in addition to the built-in file and blob storage - [storage](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-storage/graphrag_storage/tables/table_provider_factory.py) - create your own storage provider (database, etc.) beyond the file, blob, and CosmosDB ones built in diff --git a/docs/index/default_dataflow.md b/docs/index/default_dataflow.md index 9a810accb0..051c06e6eb 100644 --- a/docs/index/default_dataflow.md +++ b/docs/index/default_dataflow.md @@ -2,7 +2,7 @@ ## The GraphRAG Knowledge Model -The knowledge model is a specification for data outputs that conform to our data-model definition. You can find these definitions in the python/graphrag/graphrag/model folder within the GraphRAG repository. The following entity types are provided. The fields here represent the fields that are text-embedded by default. +The knowledge model is a specification for data outputs that conform to our data-model definition. You can find these definitions in the `packages/graphrag/graphrag/data_model` folder within the GraphRAG repository. The following entity types are provided. The fields here represent the fields that are text-embedded by default. - `Document` - An input document into the system. These either represent individual rows in a CSV or individual .txt files. - `TextUnit` - A chunk of text to analyze. The size of these chunks, their overlap, and whether they adhere to any data boundaries may be configured below. diff --git a/docs/index/inputs.md b/docs/index/inputs.md index a15f1418c1..117cf4f062 100644 --- a/docs/index/inputs.md +++ b/docs/index/inputs.md @@ -12,7 +12,7 @@ All input formats are loaded within GraphRAG and passed to the indexing pipeline | text | str | The full text of the document. | | title | str | Name of the document. Some formats allow this to be configured. | | creation_date | str | The creation date of the document, represented as an ISO8601 string. This is harvested from the source file system. | -| metadata | dict | Optional additional document metadata. More details below. | +| raw_data | dict | Optional source row/object for structured inputs. This can be used for metadata prepending during chunking. | Also see the [outputs](outputs.md) documentation for the final documents table schema saved to parquet after pipeline completion. @@ -26,7 +26,7 @@ We use an injectable InputReader provider class. This means you can implement an ## Formats -We support three file formats out-of-the-box. This covers the overwhelming majority of use cases we have encountered. If you have a different format, we recommend either implementing your own InputReader or writing a script to convert to one of these, which are widely used and supported by many tools and libraries. +We support multiple file formats out-of-the-box. If you have a different format, we recommend either implementing your own InputReader or writing a script to convert to one of these. ### Plain Text @@ -40,11 +40,23 @@ With the CSV format you can configure the `text_column`, and `title_column` if y ### JSON -JSON files (typically ending in a .json extension) contain [structured objects](https://www.json.org/). These are loaded using python's [`json.loads` method](https://docs.python.org/3/library/json.html), so your files must be properly compliant. JSON files may contain a single object in the file *or* the file may contain an array of objects at the root. We will check for and handle either of these cases. As with CSV, multiple files will be concatenated into a final table, and the `text_column` and `title_column` config options will be applied to the properties of each loaded object. Note that the specialized jsonl format produced by some libraries (one full JSON object on each line, not in an array) is not currently supported. +JSON files (typically ending in a .json extension) contain [structured objects](https://www.json.org/). These are loaded using python's [`json.loads` method](https://docs.python.org/3/library/json.html), so your files must be properly compliant. JSON files may contain a single object in the file *or* the file may contain an array of objects at the root. We will check for and handle either of these cases. As with CSV, multiple files will be concatenated into a final table, and the `text_column` and `title_column` config options will be applied to the properties of each loaded object. -## Metadata +### JSON Lines -With the structured file formats (CSV and JSON) you can configure any number of columns to be added to a persisted `metadata` field in the DataFrame. This is configured by supplying a list of column names to collect. If this is configured, the output `metadata` column will have a dict containing a key for each column, and the value of the column for that document. This metadata can optionally be used later in the GraphRAG pipeline. +JSONL files (typically ending in a .jsonl extension) are loaded one JSON object per line. Blank/whitespace-only lines are skipped. + +### Parquet + +Parquet files (typically ending in a .parquet extension) are loaded as structured rows, similar to CSV/JSON behavior for `text_column`, `title_column`, and `id_column`. + +### MarkItDown + +MarkItDown input supports document conversion via the MarkItDown reader. + +## Field Prepending + +For structured file formats, source row/object content is preserved in `raw_data`. During chunking, you can use `chunking.prepend_metadata` to prepend selected fields into each chunk. Fields can reference standard document fields (`id`, `title`, `text`, `creation_date`) or keys from `raw_data`. ### Example @@ -59,16 +71,16 @@ An early space shooter game,Space Invaders,arcade settings.yaml ```yaml -input: - metadata: [title,tag] +chunking: + prepend_metadata: [title,tag] ``` Documents DataFrame -| id | title | text | creation_date | metadata | +| id | title | text | creation_date | raw_data | | --------------------- | -------------- | --------------------------- | ----------------------------- | ---------------------------------------------- | -| (generated from text) | Hello World | My first program | (create date of software.csv) | { "title": "Hello World", "tag": "tutorial" } | -| (generated from text) | Space Invaders | An early space shooter game | (create date of software.csv) | { "title": "Space Invaders", "tag": "arcade" } | +| (generated from text) | Hello World | My first program | (create date of software.csv) | { "text": "My first program", "title": "Hello World", "tag": "tutorial" } | +| (generated from text) | Space Invaders | An early space shooter game | (create date of software.csv) | { "text": "An early space shooter game", "title": "Space Invaders", "tag": "arcade" } | ## Chunking and Metadata @@ -78,13 +90,13 @@ Imagine the following scenario: you are indexing a collection of news articles. ### Input Config -As described above, when documents are imported you can specify a list of `metadata` columns to include with each row. This must be configured for the per-chunk copying to work. +As described above, structured source fields are available via `raw_data` and may be referenced directly by field name during chunk prepending. ### Chunking Config Next, the `chunks` block needs to instruct the chunker how to handle this metadata when creating text units. By default, it is ignored. We have the following setting to include it: -- `prepend_metadata`. This instructs the importer to copy the contents of the `metadata` column for each row into the start of every single text chunk. This metadata is copied as key: value pairs on new lines. +- `prepend_metadata`. This instructs the importer to copy the selected document fields into the start of every text chunk. Values are copied as key: value pairs on new lines. ### Examples diff --git a/docs/index/outputs.md b/docs/index/outputs.md index 89a48e8526..e76ea89d59 100644 --- a/docs/index/outputs.md +++ b/docs/index/outputs.md @@ -40,7 +40,7 @@ This is the list of summarized reports for each community. | full_content | str | LM-generated full report. | | rank | float | LM-derived relevance ranking of the report based on member entity salience | rating_explanation | str | LM-derived explanation of the rank. | -| findings | dict | LM-derived list of the top 5-10 insights from the community. Contains `summary` and `explanation` values. | +| findings | dict[] | LM-derived list of the top 5-10 insights from the community. Each item contains `summary` and `explanation` values. | | full_content_json | json | Full JSON output as returned by the LM. Most fields are extracted into columns, but this JSON is sent for query summarization so we leave it to allow for prompt tuning to add fields/content by end users. | | period | str | Date of ingest, used for incremental update merges. ISO8601 | | size | int | Size of the community (entity count), used for incremental update merges. | @@ -69,7 +69,8 @@ List of document content after import. | title | str | Filename, unless otherwise configured during CSV import. | | text | str | Full text of the document. | | text_unit_ids | str[] | List of text units (chunks) that were parsed from the document. | -| metadata | dict | If specified during CSV import, this is a dict of metadata for the document. | +| creation_date | str | Creation timestamp of the source document, represented as an ISO8601 string. | +| raw_data | dict | Source row/object data loaded from structured inputs (or `null` for plain text). | ## entities List of all entities found in the data by the LM. @@ -104,5 +105,5 @@ List of all text chunks parsed from the input documents. | n_tokens | int | Number of tokens in the chunk. This should normally match the `chunk_size` config parameter, except for the last chunk which is often shorter. | | document_id | str | ID of the document the chunk came from. | | entity_ids | str[] | List of entities found in the text unit. | -| relationships_ids | str[] | List of relationships found in the text unit. | +| relationship_ids | str[] | List of relationships found in the text unit. | | covariate_ids | str[] | Optional list of covariates found in the text unit. | \ No newline at end of file From 317528686864eb8144a2363f5d9fd84a630d0186 Mon Sep 17 00:00:00 2001 From: Nathan Evans Date: Fri, 10 Jul 2026 15:59:11 -0700 Subject: [PATCH 3/3] Revert "Sync docs for schema and metadata fields" This reverts commit 55f170f7d25f58ba4494f2dee8c5690fd282c778. --- docs/config/yaml.md | 14 ++++++++----- docs/index/architecture.md | 2 +- docs/index/default_dataflow.md | 2 +- docs/index/inputs.md | 36 ++++++++++++---------------------- docs/index/outputs.md | 7 +++---- 5 files changed, 26 insertions(+), 35 deletions(-) diff --git a/docs/config/yaml.md b/docs/config/yaml.md index 7fa2a98657..c3cf870097 100644 --- a/docs/config/yaml.md +++ b/docs/config/yaml.md @@ -74,7 +74,7 @@ embedding_models: ### input -Our pipeline can ingest `.csv`, `.txt`, `.json`, `.jsonl`, `.parquet`, or MarkItDown-supported files from an input location. See the [inputs page](../index/inputs.md) for more details and examples. +Our pipeline can ingest .csv, .txt, or .json data from an input location. See the [inputs page](../index/inputs.md) for more details and examples. #### Fields @@ -86,16 +86,16 @@ Our pipeline can ingest `.csv`, `.txt`, `.json`, `.jsonl`, `.parquet`, or MarkIt - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. -- `type` **text|csv|json|jsonl|parquet|markitdown** - The type of input data to load. Default is `text` +- `type` **text|csv|json** - The type of input data to load. Default is `text` - `encoding` **str** - The encoding of the input file. Default is `utf-8` -- `file_pattern` **str** - A regex to match input files. Defaults are derived from `type` (for example `.*\.csv$`, `.*\.txt$`, `.*\.json$`, `.*\.jsonl$`, `.*\.parquet$`), but you can customize it if needed. +- `file_pattern` **str** - A regex to match input files. Default is `.*\.csv$`, `.*\.txt$`, or `.*\.json$` depending on the specified `type`, but you can customize it if needed. - `id_column` **str** - The input ID column to use. - `title_column` **str** - The input title column to use. - `text_column` **str** - The input text column to use. ### chunking -These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. You can also prepend selected document fields (standard fields like `title` or values from `raw_data`) into each chunk. +These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. Also note the `metadata` setting in the input document config, which will replicate document metadata into each chunk. #### Fields @@ -103,7 +103,7 @@ These settings configure how we parse documents into text chunks. This is necess - `encoding_model` **str** - The text encoding model to use for splitting on token boundaries. - `size` **int** - The max chunk size in tokens. - `overlap` **int** - The chunk overlap in tokens. -- `prepend_metadata` **list[str]** - Document fields to prepend on each chunk. These can be standard document fields (`id`, `title`, `text`, `creation_date`) or fields from the source row/object stored in `raw_data`. +- `prepend_metadata` **list[str]** - Metadata fields from the source document to prepend on each chunk. ## Outputs and Storage @@ -120,6 +120,8 @@ This section controls the storage mechanism used by the pipeline used for export - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. +- `type` **text|csv|json** - The type of input data to load. Default is `text` +- `encoding` **str** - The encoding of the input file. Default is `utf-8` ### update_output_storage @@ -134,6 +136,8 @@ The section defines a secondary storage location for running incremental indexin - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name. - `account_url` **str** - (blob only) The storage account blob URL to use. - `database_name` **str** - (cosmosdb only) The database name to use. +- `type` **text|csv|json** - The type of input data to load. Default is `text` +- `encoding` **str** - The encoding of the input file. Default is `utf-8` ### cache diff --git a/docs/index/architecture.md b/docs/index/architecture.md index c5d86938fe..c1c9e05f98 100644 --- a/docs/index/architecture.md +++ b/docs/index/architecture.md @@ -41,7 +41,7 @@ Several subsystems within GraphRAG use a factory pattern to register and retriev The following subsystems use a factory pattern that allows you to register your own implementations: - [language model](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-llm/graphrag_llm/completion/completion_factory.py) - implement your own `chat` and `embed` methods to use a model provider of choice beyond the built-in LiteLLM wrapper -- [input reader](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-input/graphrag_input/input_reader.py) - implement your own input document reader to support file types other than the built-in text, CSV, JSON, JSONL, Parquet, and MarkItDown readers +- [input reader](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-input/graphrag_input/input_reader.py) - implement your own input document reader to support file types other than text, CSV, and JSON - [cache](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-cache/graphrag_cache/cache_factory.py) - create your own cache storage location in addition to the file, blob, and CosmosDB ones we provide - [logger](https://github.com/microsoft/graphrag/blob/main/packages/graphrag/graphrag/logger/factory.py) - create your own log writing location in addition to the built-in file and blob storage - [storage](https://github.com/microsoft/graphrag/blob/main/packages/graphrag-storage/graphrag_storage/tables/table_provider_factory.py) - create your own storage provider (database, etc.) beyond the file, blob, and CosmosDB ones built in diff --git a/docs/index/default_dataflow.md b/docs/index/default_dataflow.md index 051c06e6eb..9a810accb0 100644 --- a/docs/index/default_dataflow.md +++ b/docs/index/default_dataflow.md @@ -2,7 +2,7 @@ ## The GraphRAG Knowledge Model -The knowledge model is a specification for data outputs that conform to our data-model definition. You can find these definitions in the `packages/graphrag/graphrag/data_model` folder within the GraphRAG repository. The following entity types are provided. The fields here represent the fields that are text-embedded by default. +The knowledge model is a specification for data outputs that conform to our data-model definition. You can find these definitions in the python/graphrag/graphrag/model folder within the GraphRAG repository. The following entity types are provided. The fields here represent the fields that are text-embedded by default. - `Document` - An input document into the system. These either represent individual rows in a CSV or individual .txt files. - `TextUnit` - A chunk of text to analyze. The size of these chunks, their overlap, and whether they adhere to any data boundaries may be configured below. diff --git a/docs/index/inputs.md b/docs/index/inputs.md index 117cf4f062..a15f1418c1 100644 --- a/docs/index/inputs.md +++ b/docs/index/inputs.md @@ -12,7 +12,7 @@ All input formats are loaded within GraphRAG and passed to the indexing pipeline | text | str | The full text of the document. | | title | str | Name of the document. Some formats allow this to be configured. | | creation_date | str | The creation date of the document, represented as an ISO8601 string. This is harvested from the source file system. | -| raw_data | dict | Optional source row/object for structured inputs. This can be used for metadata prepending during chunking. | +| metadata | dict | Optional additional document metadata. More details below. | Also see the [outputs](outputs.md) documentation for the final documents table schema saved to parquet after pipeline completion. @@ -26,7 +26,7 @@ We use an injectable InputReader provider class. This means you can implement an ## Formats -We support multiple file formats out-of-the-box. If you have a different format, we recommend either implementing your own InputReader or writing a script to convert to one of these. +We support three file formats out-of-the-box. This covers the overwhelming majority of use cases we have encountered. If you have a different format, we recommend either implementing your own InputReader or writing a script to convert to one of these, which are widely used and supported by many tools and libraries. ### Plain Text @@ -40,23 +40,11 @@ With the CSV format you can configure the `text_column`, and `title_column` if y ### JSON -JSON files (typically ending in a .json extension) contain [structured objects](https://www.json.org/). These are loaded using python's [`json.loads` method](https://docs.python.org/3/library/json.html), so your files must be properly compliant. JSON files may contain a single object in the file *or* the file may contain an array of objects at the root. We will check for and handle either of these cases. As with CSV, multiple files will be concatenated into a final table, and the `text_column` and `title_column` config options will be applied to the properties of each loaded object. +JSON files (typically ending in a .json extension) contain [structured objects](https://www.json.org/). These are loaded using python's [`json.loads` method](https://docs.python.org/3/library/json.html), so your files must be properly compliant. JSON files may contain a single object in the file *or* the file may contain an array of objects at the root. We will check for and handle either of these cases. As with CSV, multiple files will be concatenated into a final table, and the `text_column` and `title_column` config options will be applied to the properties of each loaded object. Note that the specialized jsonl format produced by some libraries (one full JSON object on each line, not in an array) is not currently supported. -### JSON Lines +## Metadata -JSONL files (typically ending in a .jsonl extension) are loaded one JSON object per line. Blank/whitespace-only lines are skipped. - -### Parquet - -Parquet files (typically ending in a .parquet extension) are loaded as structured rows, similar to CSV/JSON behavior for `text_column`, `title_column`, and `id_column`. - -### MarkItDown - -MarkItDown input supports document conversion via the MarkItDown reader. - -## Field Prepending - -For structured file formats, source row/object content is preserved in `raw_data`. During chunking, you can use `chunking.prepend_metadata` to prepend selected fields into each chunk. Fields can reference standard document fields (`id`, `title`, `text`, `creation_date`) or keys from `raw_data`. +With the structured file formats (CSV and JSON) you can configure any number of columns to be added to a persisted `metadata` field in the DataFrame. This is configured by supplying a list of column names to collect. If this is configured, the output `metadata` column will have a dict containing a key for each column, and the value of the column for that document. This metadata can optionally be used later in the GraphRAG pipeline. ### Example @@ -71,16 +59,16 @@ An early space shooter game,Space Invaders,arcade settings.yaml ```yaml -chunking: - prepend_metadata: [title,tag] +input: + metadata: [title,tag] ``` Documents DataFrame -| id | title | text | creation_date | raw_data | +| id | title | text | creation_date | metadata | | --------------------- | -------------- | --------------------------- | ----------------------------- | ---------------------------------------------- | -| (generated from text) | Hello World | My first program | (create date of software.csv) | { "text": "My first program", "title": "Hello World", "tag": "tutorial" } | -| (generated from text) | Space Invaders | An early space shooter game | (create date of software.csv) | { "text": "An early space shooter game", "title": "Space Invaders", "tag": "arcade" } | +| (generated from text) | Hello World | My first program | (create date of software.csv) | { "title": "Hello World", "tag": "tutorial" } | +| (generated from text) | Space Invaders | An early space shooter game | (create date of software.csv) | { "title": "Space Invaders", "tag": "arcade" } | ## Chunking and Metadata @@ -90,13 +78,13 @@ Imagine the following scenario: you are indexing a collection of news articles. ### Input Config -As described above, structured source fields are available via `raw_data` and may be referenced directly by field name during chunk prepending. +As described above, when documents are imported you can specify a list of `metadata` columns to include with each row. This must be configured for the per-chunk copying to work. ### Chunking Config Next, the `chunks` block needs to instruct the chunker how to handle this metadata when creating text units. By default, it is ignored. We have the following setting to include it: -- `prepend_metadata`. This instructs the importer to copy the selected document fields into the start of every text chunk. Values are copied as key: value pairs on new lines. +- `prepend_metadata`. This instructs the importer to copy the contents of the `metadata` column for each row into the start of every single text chunk. This metadata is copied as key: value pairs on new lines. ### Examples diff --git a/docs/index/outputs.md b/docs/index/outputs.md index e76ea89d59..89a48e8526 100644 --- a/docs/index/outputs.md +++ b/docs/index/outputs.md @@ -40,7 +40,7 @@ This is the list of summarized reports for each community. | full_content | str | LM-generated full report. | | rank | float | LM-derived relevance ranking of the report based on member entity salience | rating_explanation | str | LM-derived explanation of the rank. | -| findings | dict[] | LM-derived list of the top 5-10 insights from the community. Each item contains `summary` and `explanation` values. | +| findings | dict | LM-derived list of the top 5-10 insights from the community. Contains `summary` and `explanation` values. | | full_content_json | json | Full JSON output as returned by the LM. Most fields are extracted into columns, but this JSON is sent for query summarization so we leave it to allow for prompt tuning to add fields/content by end users. | | period | str | Date of ingest, used for incremental update merges. ISO8601 | | size | int | Size of the community (entity count), used for incremental update merges. | @@ -69,8 +69,7 @@ List of document content after import. | title | str | Filename, unless otherwise configured during CSV import. | | text | str | Full text of the document. | | text_unit_ids | str[] | List of text units (chunks) that were parsed from the document. | -| creation_date | str | Creation timestamp of the source document, represented as an ISO8601 string. | -| raw_data | dict | Source row/object data loaded from structured inputs (or `null` for plain text). | +| metadata | dict | If specified during CSV import, this is a dict of metadata for the document. | ## entities List of all entities found in the data by the LM. @@ -105,5 +104,5 @@ List of all text chunks parsed from the input documents. | n_tokens | int | Number of tokens in the chunk. This should normally match the `chunk_size` config parameter, except for the last chunk which is often shorter. | | document_id | str | ID of the document the chunk came from. | | entity_ids | str[] | List of entities found in the text unit. | -| relationship_ids | str[] | List of relationships found in the text unit. | +| relationships_ids | str[] | List of relationships found in the text unit. | | covariate_ids | str[] | Optional list of covariates found in the text unit. | \ No newline at end of file