Sync docs for schema and metadata fields

docs/config/yaml.md

@@ -74,7 +74,7 @@ embedding_models:
 
 ### input
 
-Our pipeline can ingest .csv, .txt, or .json data from an input location. See the [inputs page](../index/inputs.md) for more details and examples.
+Our pipeline can ingest `.csv`, `.txt`, `.json`, `.jsonl`, `.parquet`, or MarkItDown-supported files from an input location. See the [inputs page](../index/inputs.md) for more details and examples.
 
 #### Fields
 
@@ -86,24 +86,24 @@ Our pipeline can ingest .csv, .txt, or .json data from an input location. See th
   - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name.
   - `account_url` **str** - (blob only) The storage account blob URL to use.
   - `database_name` **str** - (cosmosdb only) The database name to use.
-- `type` **text|csv|json** - The type of input data to load. Default is `text`
+- `type` **text|csv|json|jsonl|parquet|markitdown** - The type of input data to load. Default is `text`
 - `encoding` **str** - The encoding of the input file. Default is `utf-8`
-- `file_pattern` **str** - A regex to match input files. Default is `.*\.csv$`, `.*\.txt$`, or `.*\.json$` depending on the specified `type`, but you can customize it if needed.
+- `file_pattern` **str** - A regex to match input files. Defaults are derived from `type` (for example `.*\.csv$`, `.*\.txt$`, `.*\.json$`, `.*\.jsonl$`, `.*\.parquet$`), but you can customize it if needed.
 - `id_column` **str** - The input ID column to use.
 - `title_column` **str** - The input title column to use.
 - `text_column` **str** - The input text column to use.
 
 ### chunking
 
-These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. Also note the `metadata` setting in the input document config, which will replicate document metadata into each chunk.
+These settings configure how we parse documents into text chunks. This is necessary because very large documents may not fit into a single context window, and graph extraction accuracy can be modulated. You can also prepend selected document fields (standard fields like `title` or values from `raw_data`) into each chunk.
 
 #### Fields
 
 - `type` **tokens|sentence** - The chunking type to use.
 - `encoding_model` **str** - The text encoding model to use for splitting on token boundaries.
 - `size` **int** - The max chunk size in tokens.
 - `overlap` **int** - The chunk overlap in tokens.
-- `prepend_metadata` **list[str]** - Metadata fields from the source document to prepend on each chunk.
+- `prepend_metadata` **list[str]** - Document fields to prepend on each chunk. These can be standard document fields (`id`, `title`, `text`, `creation_date`) or fields from the source row/object stored in `raw_data`. For structured inputs, the `raw_data` object contains any other fields present in the source file (for CSV, this is all other column data).
 
 ## Outputs and Storage
 
@@ -120,8 +120,6 @@ This section controls the storage mechanism used by the pipeline used for export
 - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name.
 - `account_url` **str** - (blob only) The storage account blob URL to use.
 - `database_name` **str** - (cosmosdb only) The database name to use.
-- `type` **text|csv|json** - The type of input data to load. Default is `text`
-- `encoding` **str** - The encoding of the input file. Default is `utf-8`
 
 ### update_output_storage
 
@@ -136,8 +134,6 @@ The section defines a secondary storage location for running incremental indexin
 - `container_name` **str** - (blob/cosmosdb only) The Azure Storage container name.
 - `account_url` **str** - (blob only) The storage account blob URL to use.
 - `database_name` **str** - (cosmosdb only) The database name to use.
-- `type` **text|csv|json** - The type of input data to load. Default is `text`
-- `encoding` **str** - The encoding of the input file. Default is `utf-8`
 
 ### cache
 

docs/examples_notebooks/drift_search.ipynb

@@ -19,14 +19,10 @@
     "import os\n",
     "\n",
     "import pandas as pd\n",
-    "from graphrag.config.enums import ModelType\n",
     "from graphrag.config.models.drift_search_config import DRIFTSearchConfig\n",
-    "from graphrag.config.models.language_model_config import LanguageModelConfig\n",
-    "from graphrag.language_model.manager import ModelManager\n",
     "from graphrag.query.indexer_adapters import (\n",
     "    read_indexer_entities,\n",
     "    read_indexer_relationships,\n",
-    "    read_indexer_report_embeddings,\n",
     "    read_indexer_reports,\n",
     "    read_indexer_text_units,\n",
     ")\n",
@@ -35,60 +31,10 @@
     ")\n",
     "from graphrag.query.structured_search.drift_search.search import DRIFTSearch\n",
     "from graphrag.tokenizer.get_tokenizer import get_tokenizer\n",
-    "from graphrag_vectors.lancedb import LanceDBVectorStore\n",
-    "\n",
-    "INPUT_DIR = \"./inputs/operation dulce\"\n",
-    "LANCEDB_URI = f\"{INPUT_DIR}/lancedb\"\n",
-    "\n",
-    "COMMUNITY_REPORT_TABLE = \"community_reports\"\n",
-    "COMMUNITY_TABLE = \"communities\"\n",
-    "ENTITY_TABLE = \"entities\"\n",
-    "RELATIONSHIP_TABLE = \"relationships\"\n",
-    "COVARIATE_TABLE = \"covariates\"\n",
-    "TEXT_UNIT_TABLE = \"text_units\"\n",
-    "COMMUNITY_LEVEL = 2\n",
-    "\n",
-    "\n",
-    "# read nodes table to get community and degree data\n",
-    "entity_df = pd.read_parquet(f\"{INPUT_DIR}/{ENTITY_TABLE}.parquet\")\n",
-    "community_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet\")\n",
-    "\n",
-    "print(f\"Entity df columns: {entity_df.columns}\")\n",
-    "\n",
-    "entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)\n",
-    "\n",
-    "# load description embeddings to an in-memory lancedb vectorstore\n",
-    "# to connect to a remote db, specify url and port values.\n",
-    "description_embedding_store = LanceDBVectorStore(\n",
-    "    db_uri=LANCEDB_URI,\n",
-    "    index_name=\"entity_description\",\n",
-    ")\n",
-    "description_embedding_store.connect()\n",
-    "\n",
-    "full_content_embedding_store = LanceDBVectorStore(\n",
-    "    db_uri=LANCEDB_URI,\n",
-    "    index_name=\"community_full_content\",\n",
-    ")\n",
-    "full_content_embedding_store.connect()\n",
-    "\n",
-    "print(f\"Entity count: {len(entity_df)}\")\n",
-    "entity_df.head()\n",
-    "\n",
-    "relationship_df = pd.read_parquet(f\"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet\")\n",
-    "relationships = read_indexer_relationships(relationship_df)\n",
-    "\n",
-    "print(f\"Relationship count: {len(relationship_df)}\")\n",
-    "relationship_df.head()\n",
-    "\n",
-    "text_unit_df = pd.read_parquet(f\"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet\")\n",
-    "text_units = read_indexer_text_units(text_unit_df)\n",
-    "\n",
-    "print(f\"Text unit records: {len(text_unit_df)}\")\n",
-    "text_unit_df.head()\n",
-    "\n",
-    "report_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet\")\n",
-    "reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)\n",
-    "read_indexer_report_embeddings(reports, full_content_embedding_store)"
+    "from graphrag_llm.completion import create_completion\n",
+    "from graphrag_llm.config import ModelConfig\n",
+    "from graphrag_llm.embedding import create_embedding\n",
+    "from graphrag_vectors import IndexSchema, LanceDBVectorStore"
    ]
   },
   {
@@ -99,33 +45,61 @@
    "source": [
     "api_key = os.environ[\"GRAPHRAG_API_KEY\"]\n",
     "\n",
-    "chat_config = LanguageModelConfig(\n",
-    "    api_key=api_key,\n",
-    "    type=ModelType.Chat,\n",
+    "chat_config = ModelConfig(\n",
+    "    type=\"litellm\",\n",
     "    model_provider=\"openai\",\n",
     "    model=\"gpt-4.1\",\n",
-    "    max_retries=20,\n",
-    ")\n",
-    "chat_model = ModelManager().get_or_create_chat_model(\n",
-    "    name=\"local_search\",\n",
-    "    model_type=ModelType.Chat,\n",
-    "    config=chat_config,\n",
+    "    api_key=api_key,\n",
     ")\n",
+    "chat_model = create_completion(chat_config)\n",
     "\n",
     "tokenizer = get_tokenizer(chat_config)\n",
     "\n",
-    "embedding_config = LanguageModelConfig(\n",
-    "    api_key=api_key,\n",
-    "    type=ModelType.Embedding,\n",
+    "embedding_config = ModelConfig(\n",
+    "    type=\"litellm\",\n",
     "    model_provider=\"openai\",\n",
     "    model=\"text-embedding-3-large\",\n",
-    "    max_retries=20,\n",
+    "    api_key=api_key,\n",
+    ")\n",
+    "\n",
+    "text_embedder = create_embedding(embedding_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# parquet files generated from indexing pipeline\n",
+    "INPUT_DIR = \"./inputs/operation dulce\"\n",
+    "LANCEDB_URI = \"./lancedb\"\n",
+    "COMMUNITY_TABLE = \"communities\"\n",
+    "COMMUNITY_REPORT_TABLE = \"community_reports\"\n",
+    "ENTITY_TABLE = \"entities\"\n",
+    "RELATIONSHIP_TABLE = \"relationships\"\n",
+    "TEXT_UNIT_TABLE = \"text_units\"\n",
+    "COMMUNITY_LEVEL = 2\n",
+    "\n",
+    "community_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_TABLE}.parquet\")\n",
+    "report_df = pd.read_parquet(f\"{INPUT_DIR}/{COMMUNITY_REPORT_TABLE}.parquet\")\n",
+    "entity_df = pd.read_parquet(f\"{INPUT_DIR}/{ENTITY_TABLE}.parquet\")\n",
+    "relationship_df = pd.read_parquet(f\"{INPUT_DIR}/{RELATIONSHIP_TABLE}.parquet\")\n",
+    "text_unit_df = pd.read_parquet(f\"{INPUT_DIR}/{TEXT_UNIT_TABLE}.parquet\")\n",
+    "\n",
+    "reports = read_indexer_reports(report_df, community_df, COMMUNITY_LEVEL)\n",
+    "entities = read_indexer_entities(entity_df, community_df, COMMUNITY_LEVEL)\n",
+    "relationships = read_indexer_relationships(relationship_df)\n",
+    "text_units = read_indexer_text_units(text_unit_df)\n",
+    "\n",
+    "# Connect to the existing GraphRAG vector index for entity-description embeddings.\n",
+    "description_embedding_store = LanceDBVectorStore(\n",
+    "    index_schema=IndexSchema(index_name=\"default-entity-description\")\n",
     ")\n",
+    "description_embedding_store.connect(db_uri=LANCEDB_URI)\n",
     "\n",
-    "text_embedder = ModelManager().get_or_create_embedding_model(\n",
-    "    name=\"local_search_embedding\",\n",
-    "    model_type=ModelType.Embedding,\n",
-    "    config=embedding_config,\n",
+    "print(\n",
+    "    f\"Loaded reports={len(reports)}, entities={len(entities)}, relationships={len(relationships)}, text_units={len(text_units)}\"\n",
     ")"
    ]
   },
@@ -188,7 +162,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": "graphrag-monorepo (3.12.10)",
    "language": "python",
    "name": "python3"
   },