diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7..66052b2a 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -26,9 +26,16 @@ def _table_name_from_csv(filename: str) -> str: print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: df = pd.read_csv(f, index_col="ROW_ID") df.to_sql(table, CONNECTION_STRING)