From fe0d71bf1ebdcee8f31413902814e75f627ea579 Mon Sep 17 00:00:00 2001
From: Taksh <takshkothari09@gmail.com>
Date: Sun, 19 Jul 2026 11:14:22 +0300
Subject: [PATCH] fix: pin sqlite import dtypes for mixed columns

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 mimic-iii/buildmimic/sqlite/import.py | 38 +++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py
index 801651c7..fb565b36 100644
--- a/mimic-iii/buildmimic/sqlite/import.py
+++ b/mimic-iii/buildmimic/sqlite/import.py
@@ -10,6 +10,28 @@
 CHUNKSIZE = 10 ** 6
 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME)
 
+# Column dtypes for tables that trigger pandas mixed-type warnings when
+# loaded with the default low_memory chunked inference (see #1237).
+# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql.
+TABLE_DTYPES = {
+    "datetimeevents": {
+        "WARNING": "Int64",
+        "ERROR": "Int64",
+        "RESULTSTATUS": "string",
+        "STOPPED": "string",
+    },
+    "inputevents_cv": {
+        "ORIGINALRATE": "float64",
+        "ORIGINALRATEUOM": "string",
+        "ORIGINALSITE": "string",
+    },
+    "noteevents": {
+        "CHARTTIME": "string",
+        "STORETIME": "string",
+        "ISERROR": "string",
+    },
+}
+
 
 def _table_name_from_csv(filename: str) -> str:
     """Derive SQL table name from a CSV path (literal suffix, not str.strip)."""
@@ -21,6 +43,18 @@ def _table_name_from_csv(filename: str) -> str:
     return name.lower()
 
 
+def _read_csv(path, table, **kwargs):
+    # low_memory=False avoids dtype re-inference across chunks; table-specific
+    # dtypes pin the columns that otherwise flip between numeric/object.
+    return pd.read_csv(
+        path,
+        index_col="ROW_ID",
+        low_memory=False,
+        dtype=TABLE_DTYPES.get(table),
+        **kwargs,
+    )
+
+
 if os.path.exists(DATABASE_NAME):
     msg = "File {} already exists.".format(DATABASE_NAME)
     print(msg)
@@ -30,11 +64,11 @@ def _table_name_from_csv(filename: str) -> str:
     print("Starting processing {}".format(f))
     table = _table_name_from_csv(f)
     if os.path.getsize(f) < THRESHOLD_SIZE:
-        df = pd.read_csv(f, index_col="ROW_ID")
+        df = _read_csv(f, table)
         df.to_sql(table, CONNECTION_STRING)
     else:
         # If the file is too large, let's do the work in chunks
-        for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
+        for chunk in _read_csv(f, table, chunksize=CHUNKSIZE):
             chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
     print("Finished processing {}".format(f))