From c06aeec351bafa49e88006a03b77ba3a1f04642e Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:22:58 +0300 Subject: [PATCH 01/47] fix: SECOND-based DATETIME_DIFF in urine_output_rate Match kdigo_uo so BigQuery truncation and Postgres fractional diffs no longer diverge on the 6/12h rate windows (#1549). Co-authored-by: Cursor --- .../measurement/urine_output_rate.sql | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 01939211..bc674014 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -16,12 +16,14 @@ WITH tm AS ( ) -- now calculate time since last UO measurement +-- Use SECOND diffs (like kdigo_uo) so BigQuery and PostgreSQL agree (#1549). +-- BigQuery DATETIME_DIFF(..., MINUTE/HOUR) truncates; SECOND keeps fractions. , uo_tm AS ( SELECT tm.stay_id , CASE WHEN LAG(charttime) OVER w IS NULL - THEN DATETIME_DIFF(charttime, intime_hr, MINUTE) - ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, MINUTE) + THEN DATETIME_DIFF(charttime, intime_hr, SECOND) / 60.0 + ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, SECOND) / 60.0 END AS tm_since_last_uo , uo.charttime , uo.urineoutput @@ -45,24 +47,33 @@ WITH tm AS ( -- to 1 hour of UO, therefore we use '5' and '11' to restrict the -- period, rather than 6/12 this assumption may overestimate UO rate -- when documentation is done less than hourly - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr , ROUND(CAST( SUM( CASE - WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_6hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr , ROUND(CAST( SUM( CASE - WHEN - DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_12hr From 5c49bc45ce9670917d55637ad5ffbddbe29df991 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:23:45 +0300 Subject: [PATCH 02/47] fix: escape password and quote ids in create_mimic_user Passwords with quotes and unusual user/db names broke the CREATE USER / DATABASE statements. Co-authored-by: Cursor --- .../buildmimic/postgres/create_mimic_user.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce..d639f5fc 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi From d4a83056737ed70ea2bacc5776d95796cb217828 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:24:47 +0300 Subject: [PATCH 03/47] fix: pin sqlite import dtypes for mixed columns Stop pandas DtypeWarning on chartevents/datetimeevents/ inputevents_cv/noteevents during chunked import (#1237). Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.py | 44 +++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7..dc898b14 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,6 +49,18 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) @@ -30,11 +70,11 @@ def _table_name_from_csv(filename: str) -> str: print("Starting processing {}".format(f)) table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) From b03585805990a1d21d5776edfcce10eae696847d Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:29:02 +0300 Subject: [PATCH 04/47] fix: avoid escaped quote in neuroblock_dose stopped check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sqlglot rejects 'D/C''d' as adjacent string literals, breaking BigQuery→postgres/duckdb transpile of this concept. Co-authored-by: Cursor --- mimic-iii/concepts/durations/neuroblock_dose.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index 098b557e..3024d340 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -28,7 +28,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped -- we only include continuous infusions, therefore expect a rate , max(case @@ -68,7 +68,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped , max(case when valuenum <= 10 then 0 else 1 end) as drug_null -- educated guess! From 24c02b4623b057a34ce213a3f33c843b28c69ea3 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:46:57 +0300 Subject: [PATCH 05/47] fix: use DATETIME_DIFF for neuroblock gap check BigQuery rejects timestamp subtraction with interval compares; use DATETIME_DIFF(..., HOUR) like the rest of the concepts. Co-authored-by: Cursor --- mimic-iii/concepts/durations/neuroblock_dose.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index 3024d340..a1ce5c9a 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -179,7 +179,11 @@ select ) = 1 then 1 - when (CHARTTIME - (LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime))) > (interval '8 hours') then 1 + when DATETIME_DIFF( + CHARTTIME, + LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime), + HOUR + ) > 8 then 1 else null end as drug_start From 23f708457ef7a874f32adc4349837e38f9628447 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 12:53:32 +0300 Subject: [PATCH 06/47] fix: check $1 not PID in copy_concepts script $$1 expands to the shell PID, so the version arg was never validated. Co-authored-by: Cursor --- mimic-iv/concepts/copy_concepts_to_versioned_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh index df0afa8f..547b64a4 100644 --- a/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh +++ b/mimic-iv/concepts/copy_concepts_to_versioned_schema.sh @@ -1,6 +1,6 @@ #!/bin/bash # This script copies the concepts in the BigQuery table mimiciv_derived to mimiciv_${VERSION}_derived. -if [ -z "$$1" ]; then +if [ -z "$1" ]; then echo "Usage: $0 " exit 1 fi From cb18187432c059dc23c7a933af57d22a3c9e55fa Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:02:02 +0300 Subject: [PATCH 07/47] ci: dont fail sqlfluff job when annotate lacks fork perms Lint already ran; fork tokens cannot create check runs. Co-authored-by: Cursor --- .github/workflows/lint_sqlfluff.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 9401198e..ca4c8c8b 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file From e560823c2111a7c79b148eeacd8aedc98733c02f Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:35:16 +0300 Subject: [PATCH 08/47] fix: load .csv and .csv.gz in duckdb imports Co-authored-by: Cursor --- .../buildmimic/duckdb/import_duckdb.sh | 17 +++++++++++++--- .../buildmimic/duckdb/import_duckdb.sh | 17 +++++++++++++--- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 20 ++++++++++++++++--- 3 files changed, 45 insertions(+), 9 deletions(-) diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 9142f7b4..ca408fdb 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -93,7 +93,11 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed @@ -102,9 +106,16 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index c0425fe6..d9ae09a3 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -96,7 +96,11 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note @@ -105,9 +109,10 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -122,5 +127,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ab1d37fa..23c58d9f 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -101,7 +101,13 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv @@ -110,9 +116,11 @@ find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do (hosp|icu) ;; # OK (*) continue; esac + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. \c" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -127,5 +135,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." From a2d01c4e6dd5b79395cc332813859c0013f951c7 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:35:16 +0300 Subject: [PATCH 09/47] fix: require epi/norepi rate > 0 for SOFA CV score 3 Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/sofa.sql | 2 +- mimic-iv/concepts/firstday/first_day_sofa.sql | 4 ++-- mimic-iv/concepts/score/sofa.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mimic-iii/concepts/severityscores/sofa.sql b/mimic-iii/concepts/severityscores/sofa.sql index 9afc5be1..52cb8d5f 100644 --- a/mimic-iii/concepts/severityscores/sofa.sql +++ b/mimic-iii/concepts/severityscores/sofa.sql @@ -218,7 +218,7 @@ left join `physionet-data.mimiciii_derived.gcs_first_day` gcs -- Cardiovascular , case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iv/concepts/firstday/first_day_sofa.sql b/mimic-iv/concepts/firstday/first_day_sofa.sql index 62282af7..ffd3d3c2 100644 --- a/mimic-iv/concepts/firstday/first_day_sofa.sql +++ b/mimic-iv/concepts/firstday/first_day_sofa.sql @@ -196,8 +196,8 @@ WITH vaso_stg AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 THEN 2 WHEN mbp_min < 70 THEN 1 diff --git a/mimic-iv/concepts/score/sofa.sql b/mimic-iv/concepts/score/sofa.sql index 0dbf28d1..049b3920 100644 --- a/mimic-iv/concepts/score/sofa.sql +++ b/mimic-iv/concepts/score/sofa.sql @@ -280,8 +280,8 @@ WITH co AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 From 1cfb063208b6a401ec7dab31b01974a1f917a3df Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 10:40:47 +0300 Subject: [PATCH 10/47] fix: quote datadir for postgres load and wget check already handles spaces (#481/#2068); load still used unquoted datadir and \cd without quoting, so paths with spaces broke at import. Co-authored-by: Cursor --- mimic-iii/buildmimic/postgres/Makefile | 8 ++++---- mimic-iii/buildmimic/postgres/postgres_load_data.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 17ea42e2..29953adb 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data.sql b/mimic-iii/buildmimic/postgres/postgres_load_data.sql index 3fe94c5e..5245bac8 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql index c47799b5..36c27dca 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql index e993635e..4cf2835c 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; From 81e415508fdb19d38fd6303d86726e9b662abb5c Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:26:34 +0300 Subject: [PATCH 11/47] fix: write transpiled sql as utf-8 --- src/mimic_utils/transpile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541..29924c18 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) From b2711aa742540f4d2fe8939e9a165cebce9de69f Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:30:39 +0300 Subject: [PATCH 12/47] fix: exact table name match in validate_concepts.sh --- mimic-iv/concepts/validate_concepts.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index eeaddb82..645a82a3 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,7 +32,8 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then + # exact field match (regex grep can false-hit prefixes) + if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -47,7 +48,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then + if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 From 8b8833a27a3c406d94cac2ae563f753a0efe0f12 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 13:55:05 +0300 Subject: [PATCH 13/47] fix: exclude ICD-10-CM C4A from Charlson malignancy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C4A (Merkel) sits in the C45–C58 string range; Quan 2005 uses WHO ICD-10 and excludes skin malignancy (#2017). Co-authored-by: Cursor --- mimic-iv/concepts/comorbidity/charlson.sql | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mimic-iv/concepts/comorbidity/charlson.sql b/mimic-iv/concepts/comorbidity/charlson.sql index 15911b37..2546e7c8 100644 --- a/mimic-iv/concepts/comorbidity/charlson.sql +++ b/mimic-iv/concepts/comorbidity/charlson.sql @@ -285,6 +285,8 @@ WITH diag AS ( -- Any malignancy, including lymphoma and leukemia, -- except malignant neoplasm of skin. + -- Exclude ICD-10-CM-only C4A (Merkel); Quan 2005 is WHO ICD-10 (#2017). + -- C7A/C7B sit outside these ranges and are also left excluded. , MAX(CASE WHEN SUBSTR(icd9_code, 1, 3) BETWEEN '140' AND '172' OR @@ -302,7 +304,10 @@ WITH diag AS ( OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C37' AND 'C41' OR - SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' + ( + SUBSTR(icd10_code, 1, 3) BETWEEN 'C45' AND 'C58' + AND SUBSTR(icd10_code, 1, 3) != 'C4A' + ) OR SUBSTR(icd10_code, 1, 3) BETWEEN 'C60' AND 'C76' OR From 3327400c1f51ef1ffca3287c2fe7f584d707bc20 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:08:41 +0300 Subject: [PATCH 14/47] fix: include provider/caregiver/ingredientevents in demo validate Demo loads these tables but validate_demo omitted them, so incomplete demo imports could still pass. Co-authored-by: Cursor --- mimic-iv/buildmimic/mysql/validate_demo.sql | 6 ++++++ mimic-iv/buildmimic/postgres/validate_demo.sql | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e8413..ebf4f02d 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e..049497e5 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) From 475bbda9e62229159bcb4c9c77ac7c87dbad254d Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 12:36:07 +0300 Subject: [PATCH 15/47] fix: use globs instead of ls in make_concepts.sh --- mimic-iv/concepts/make_concepts.sh | 45 +++++++++++++++--------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc2..cc633fc3 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -44,7 +44,7 @@ for table_path in demographics/icustay_times; do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,30 +54,29 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "$d"/*.sql; do - # only run SQL queries - if [[ "${fn: -4}" == ".sql" ]]; then - # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + # table name is file name minus extension + tbl="${fn%.sql}" - # skip certain tables where order matters - skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 - do - if [[ "${tbl}" == "${skip_table}" ]]; then - skip=1 - break - fi - done; - if [[ "${skip}" == "1" ]]; then - continue - fi - - # not skipping - so generate the table on bigquery - echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + # skip certain tables where order matters + skip=0 + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 + do + if [[ "${tbl}" == "${skip_table}" ]]; then + skip=1 + break + fi + done; + if [[ "${skip}" == "1" ]]; then + continue fi + + # not skipping - so generate the table on bigquery + echo "Generating ${TARGET_DATASET}.${tbl}" + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" done done @@ -88,5 +87,5 @@ do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done From eafee6f5b3ceb5e1c3e9c4d9bb387cba2dda8c8c Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:08:41 +0300 Subject: [PATCH 16/47] fix: escape quotes in mimic-iii duckdb COPY paths Paths with apostrophes broke COPY string literals. Co-authored-by: Cursor --- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 15d4977f..7ccf9e2d 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -84,9 +84,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. \c" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." From 51bd9d439e1c4343ef92f6bf465b580afa808a60 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 11:22:20 +0300 Subject: [PATCH 17/47] fix: quote sqlite import.sh paths for spaces Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.sh | 6 +++--- mimic-iv/buildmimic/sqlite/import.sh | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce..58b85abb 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE < Date: Thu, 23 Jul 2026 14:15:02 +0300 Subject: [PATCH 18/47] fix: load plain .csv in mimic-iii sqlite import.py import.sh already accepts both; import.py only globbed *.csv.gz. Co-authored-by: Cursor --- mimic-iii/buildmimic/sqlite/import.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index dc898b14..4e0113d3 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -66,9 +66,16 @@ def _read_csv(path, table, **kwargs): print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) From f7c8a45b413128cb5c16428952f548f8c49a27d1 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:15:02 +0300 Subject: [PATCH 19/47] fix: quote DIRNAME in duckdb import case patterns Unquoted case breaks when the parent folder name has spaces. Co-authored-by: Cursor --- mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv-note/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index ca408fdb..f6e6a2da 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -102,7 +102,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index d9ae09a3..487d7405 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -105,7 +105,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index 23c58d9f..ee37c5a1 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -112,7 +112,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac From b3f8c6676dbc6fdb103bf2e0dddc2900ef7e8734 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 19:48:41 +0300 Subject: [PATCH 20/47] fix: quote paths in mimic-iii make-concepts.sh --- mimic-iii/concepts/make-concepts.sh | 140 ++++++++++++++-------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index a7d1152c..01e15f00 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -14,108 +14,108 @@ echo '' set -x echo 'Top level files..' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.code_status < code_status.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.echo_data < echo_data.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.code_status" < "code_status.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.echo_data" < "echo_data.sql" echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.heightweight < demographics/heightweight.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.icustay_detail < demographics/icustay_detail.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.heightweight" < "demographics/heightweight.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_detail" < "demographics/icustay_detail.sql" # Durations (usually of treatments) echo 'Directory 2: durations' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_classification < durations/ventilation_classification.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_durations < durations/ventilation_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.crrt_durations < durations/crrt_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.adenosine_durations < durations/adenosine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_durations < durations/dobutamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_durations < durations/dopamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_durations < durations/epinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.isuprel_durations < durations/isuprel_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.milrinone_durations < durations/milrinone_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_durations < durations/norepinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_durations < durations/phenylephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_durations < durations/vasopressin_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressor_durations < durations/vasopressor_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_durations < durations/weight_durations.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_classification" < "durations/ventilation_classification.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_durations" < "durations/ventilation_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.crrt_durations" < "durations/crrt_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.adenosine_durations" < "durations/adenosine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_durations" < "durations/dobutamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_durations" < "durations/dopamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_durations" < "durations/epinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.isuprel_durations" < "durations/isuprel_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.milrinone_durations" < "durations/milrinone_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_durations" < "durations/norepinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_durations" < "durations/phenylephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_durations" < "durations/vasopressin_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressor_durations" < "durations/vasopressor_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_durations" < "durations/weight_durations.sql" # dose queries for vasopressors -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_dose < durations/dobutamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_dose < durations/dopamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_dose < durations/epinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_dose < durations/norepinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_dose < durations/phenylephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_dose < durations/vasopressin_dose.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_dose" < "durations/dobutamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_dose" < "durations/dopamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_dose" < "durations/epinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_dose" < "durations/norepinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_dose" < "durations/phenylephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_dose" < "durations/vasopressin_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_vital < pivot/pivoted_vital.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_uo < pivot/pivoted_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_rrt < pivot/pivoted_rrt.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_lab < pivot/pivoted_lab.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_icp < pivot/pivoted_icp.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_height < pivot/pivoted_height.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_gcs < pivot/pivoted_gcs.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_fio2 < pivot/pivoted_fio2.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg < pivot/pivoted_bg.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_vital" < "pivot/pivoted_vital.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_uo" < "pivot/pivoted_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_rrt" < "pivot/pivoted_rrt.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_lab" < "pivot/pivoted_lab.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_invasive_lines" < "pivot/pivoted_invasive_lines.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_icp" < "pivot/pivoted_icp.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_height" < "pivot/pivoted_height.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_gcs" < "pivot/pivoted_gcs.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_fio2" < "pivot/pivoted_fio2.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg" < "pivot/pivoted_bg.sql" # pivoted_bg_art must be run after pivoted_bg -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg_art < pivot/pivoted_bg_art.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg_art" < "pivot/pivoted_bg_art.sql" # pivoted oasis depends on icustay_hours in demographics -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_oasis < pivot/pivoted_oasis.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_oasis" < "pivot/pivoted_oasis.sql" # pivoted sofa depends on many above pivoted views, ventilation_durations, and dose queries -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_sofa < pivot/pivoted_sofa.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_sofa" < "pivot/pivoted_sofa.sql" echo 'Directory 4: comorbidity' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_quan < comorbidity/elixhauser_quan.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37" < "comorbidity/elixhauser_ahrq_v37.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg" < "comorbidity/elixhauser_ahrq_v37_no_drg.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_quan" < "comorbidity/elixhauser_quan.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_ahrq" < "comorbidity/elixhauser_score_ahrq.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_quan" < "comorbidity/elixhauser_score_quan.sql" echo 'Directory 5: firstday' # data which is extracted from a patient's first ICU stay -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day < firstday/blood_gas_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.gcs_first_day < firstday/gcs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.labs_first_day < firstday/labs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.rrt_first_day < firstday/rrt_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output_first_day < firstday/urine_output_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_first_day < firstday/ventilation_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vitals_first_day < firstday/vitals_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_first_day < firstday/weight_first_day.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day" < "firstday/blood_gas_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day_arterial" < "firstday/blood_gas_first_day_arterial.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.gcs_first_day" < "firstday/gcs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.labs_first_day" < "firstday/labs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.rrt_first_day" < "firstday/rrt_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output_first_day" < "firstday/urine_output_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_first_day" < "firstday/ventilation_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vitals_first_day" < "firstday/vitals_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_first_day" < "firstday/weight_first_day.sql" echo 'Directory 6: fluid_balance' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output < fluid_balance/urine_output.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output" < "fluid_balance/urine_output.sql" echo 'Directory 7: sepsis' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.angus < sepsis/angus.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.martin < sepsis/martin.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.explicit < sepsis/explicit.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.angus" < "sepsis/angus.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.martin" < "sepsis/martin.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.explicit" < "sepsis/explicit.sql" # diagnosis mapping using CCS echo 'Directory 8: diagnosis' # load the ccs_multi_dx.csv.gz file into bq -bq load --source_format=CSV ${TARGET_DATASET}.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ccs_dx < diagnosis/ccs_dx.sql +bq load --source_format=CSV "${TARGET_DATASET}.ccs_multi_dx" "diagnosis/ccs_multi_dx.csv.gz" "diagnosis/ccs_multi_dx.json" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ccs_dx" < "diagnosis/ccs_dx.sql" # Organ failure scores echo 'Directory 9: organfailure' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_creatinine < organfailure/kdigo_creatinine.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_uo < organfailure/kdigo_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages < organfailure/kdigo_stages.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_7day < organfailure/kdigo_stages_7day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.meld < organfailure/meld.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_creatinine" < "organfailure/kdigo_creatinine.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_uo" < "organfailure/kdigo_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages" < "organfailure/kdigo_stages.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_7day" < "organfailure/kdigo_stages_7day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_48hr" < "organfailure/kdigo_stages_48hr.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.meld" < "organfailure/meld.sql" # Severity of illness scores (requires many views from above) echo 'Directory 10: severityscores' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.oasis < severityscores/oasis.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sofa < severityscores/sofa.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.saps < severityscores/saps.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sapsii < severityscores/sapsii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.apsiii < severityscores/apsiii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.lods < severityscores/lods.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sirs < severityscores/sirs.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.oasis" < "severityscores/oasis.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sofa" < "severityscores/sofa.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.saps" < "severityscores/saps.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sapsii" < "severityscores/sapsii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.apsiii" < "severityscores/apsiii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.lods" < "severityscores/lods.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sirs" < "severityscores/sirs.sql" echo 'Finished creating concepts.' \ No newline at end of file From dd56cd399054b801abf391778c213c96e62114c5 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 11:54:23 +0300 Subject: [PATCH 21/47] fix: low_memory=False on mimic-iv sqlite read_csv --- mimic-iv/buildmimic/sqlite/import.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d43647..31b880c4 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) From 6f9950994b4ac0f642ddde612b8b23cb9fb31ef9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:19:38 +0300 Subject: [PATCH 22/47] fix: use printf for duckdb load progress echo .. \c prints a literal \c on bash; printf is portable. Co-authored-by: Cursor --- mimic-iii/buildmimic/duckdb/import_duckdb.sh | 2 +- mimic-iv/buildmimic/duckdb/import_duckdb.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 7ccf9e2d..badf83b8 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -86,7 +86,7 @@ find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" # Escape single quotes for SQL string literal FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") - echo "Loading $FILE .. \c" + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ee37c5a1..32f608c4 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -118,7 +118,7 @@ find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while esac # Escape single quotes for SQL string literal FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") - echo "Loading $FILE .. \c" + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL From f8590e83653bb9b807b2239536f4785de8d405f0 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:21:30 +0300 Subject: [PATCH 23/47] fix: use globs and quote paths in cxr label scripts --- mimic-iv-cxr/txt/negbio/run_negbio.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 13cba479..d58e7fbd 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,20 +32,22 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn in `ls $BASE_FOLDER`; +for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; then - export INPUT_FILE=${BASE_FOLDER}/$fn + export INPUT_FILE=$fn_path # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} echo $OUTPUT_DIR - running NegBio.. - python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE + python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 From cde84eb873dd624ac5b9ce5751c3fb4751fc7eb9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Sun, 19 Jul 2026 16:21:51 +0300 Subject: [PATCH 24/47] fix: glob + quote paths in chexpert runner too --- .../txt/chexpert/run_chexpert_on_files.sh | 21 +++++++++++++------ mimic-iv-cxr/txt/negbio/run_negbio.sh | 4 ++-- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index 52e34e0f..f996541a 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,11 +22,20 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn in `ls $REPORT_PATH`; do - echo `date`: $fn - fn_stem=`echo $fn | cut -d. -f 1` +for fn_path in "$REPORT_PATH"/*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + echo "$(date): $fn" + fn_stem=${fn%.csv} # run chexpert - must be run from chexpert folder - python $CHEXPERT_PATH/label.py --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt - echo `date`: done! + python "$CHEXPERT_PATH/label.py" --verbose \ + --reports_path "$fn_path" \ + --output_path "${fn_stem}_labeled.csv" \ + --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \ + --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \ + --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \ + --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \ + --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + echo "$(date): done!" echo '' -done \ No newline at end of file +done diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index d58e7fbd..dab200cd 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -46,7 +46,7 @@ do # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} - echo $OUTPUT_DIR - running NegBio.. + echo "$OUTPUT_DIR - running NegBio.." python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 @@ -57,7 +57,7 @@ do # ultimate filename we save the labels to export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv - python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/* + python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done echo "Done looping through files." From e7d5561d7ddb0f56cd5d7f86667d7c2c327d78ac Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:26:00 +0300 Subject: [PATCH 25/47] fix: quote mimic_data_dir for postgres \\cd in IV/ED/Note Same path-with-spaces break as #481; use :'mimic_data_dir'. Co-authored-by: Cursor --- mimic-iv-ed/buildmimic/postgres/load.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv-note/buildmimic/postgres/load.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv/buildmimic/postgres/load.sql | 2 +- mimic-iv/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv/buildmimic/postgres/load_gz.sql | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mimic-iv-ed/buildmimic/postgres/load.sql b/mimic-iv-ed/buildmimic/postgres/load.sql index 81fc9b2a..5b7e50ca 100644 --- a/mimic-iv-ed/buildmimic/postgres/load.sql +++ b/mimic-iv-ed/buildmimic/postgres/load.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_7z.sql b/mimic-iv-ed/buildmimic/postgres/load_7z.sql index 15d031f9..b5473361 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_7z.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_7z.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_gz.sql b/mimic-iv-ed/buildmimic/postgres/load_gz.sql index 6e92a132..0d42e5d1 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_gz.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -- Change to the directory containing the data files -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-note/buildmimic/postgres/load.sql b/mimic-iv-note/buildmimic/postgres/load.sql index 38a79a82..27e73761 100644 --- a/mimic-iv-note/buildmimic/postgres/load.sql +++ b/mimic-iv-note/buildmimic/postgres/load.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_7z.sql b/mimic-iv-note/buildmimic/postgres/load_7z.sql index 57c4dee6..2d02f598 100644 --- a/mimic-iv-note/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-note/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_gz.sql b/mimic-iv-note/buildmimic/postgres/load_gz.sql index bbe9ba64..8b32d85d 100644 --- a/mimic-iv-note/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-note/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load.sql b/mimic-iv/buildmimic/postgres/load.sql index e48848d9..c2c2f52f 100644 --- a/mimic-iv/buildmimic/postgres/load.sql +++ b/mimic-iv/buildmimic/postgres/load.sql @@ -5,7 +5,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- The script assumes the files are in the hosp and icu subfolders of mimic_data_dir -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure correct encoding is defined as -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_7z.sql b/mimic-iv/buildmimic/postgres/load_7z.sql index 5f238bc4..8ccc4604 100644 --- a/mimic-iv/buildmimic/postgres/load_7z.sql +++ b/mimic-iv/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_gz.sql b/mimic-iv/buildmimic/postgres/load_gz.sql index f77ac82f..e9a895a4 100644 --- a/mimic-iv/buildmimic/postgres/load_gz.sql +++ b/mimic-iv/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :mimic_data_dir +\cd :'mimic_data_dir' -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; From 6b3785640cc87ca8f36b65a689e7b0eb3d37f2bf Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:30:20 +0300 Subject: [PATCH 26/47] revert: unquote psql \\cd mimic_data_dir :'var' makes \\cd include literal quotes in the path and breaks CI. Co-authored-by: Cursor --- mimic-iii/buildmimic/postgres/postgres_load_data.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql | 2 +- mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-ed/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv-note/buildmimic/postgres/load.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv-note/buildmimic/postgres/load_gz.sql | 2 +- mimic-iv/buildmimic/postgres/load.sql | 2 +- mimic-iv/buildmimic/postgres/load_7z.sql | 2 +- mimic-iv/buildmimic/postgres/load_gz.sql | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data.sql b/mimic-iii/buildmimic/postgres/postgres_load_data.sql index 5245bac8..3fe94c5e 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql index 36c27dca..c47799b5 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_7zip.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql index 4cf2835c..e993635e 100644 --- a/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql +++ b/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql @@ -9,7 +9,7 @@ -------------------------------------------------------- -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: -- SET search_path TO mimiciii; diff --git a/mimic-iv-ed/buildmimic/postgres/load.sql b/mimic-iv-ed/buildmimic/postgres/load.sql index 5b7e50ca..81fc9b2a 100644 --- a/mimic-iv-ed/buildmimic/postgres/load.sql +++ b/mimic-iv-ed/buildmimic/postgres/load.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_7z.sql b/mimic-iv-ed/buildmimic/postgres/load_7z.sql index b5473361..15d031f9 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_7z.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_7z.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-ed/buildmimic/postgres/load_gz.sql b/mimic-iv-ed/buildmimic/postgres/load_gz.sql index 0d42e5d1..6e92a132 100644 --- a/mimic-iv-ed/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-ed/buildmimic/postgres/load_gz.sql @@ -12,7 +12,7 @@ -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -- Change to the directory containing the data files -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- If running scripts individually, you can set the schema where all tables are created as follows: SET search_path TO mimiciv_ed; diff --git a/mimic-iv-note/buildmimic/postgres/load.sql b/mimic-iv-note/buildmimic/postgres/load.sql index 27e73761..38a79a82 100644 --- a/mimic-iv-note/buildmimic/postgres/load.sql +++ b/mimic-iv-note/buildmimic/postgres/load.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_7z.sql b/mimic-iv-note/buildmimic/postgres/load_7z.sql index 2d02f598..57c4dee6 100644 --- a/mimic-iv-note/buildmimic/postgres/load_7z.sql +++ b/mimic-iv-note/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv-note/buildmimic/postgres/load_gz.sql b/mimic-iv-note/buildmimic/postgres/load_gz.sql index 8b32d85d..bbe9ba64 100644 --- a/mimic-iv-note/buildmimic/postgres/load_gz.sql +++ b/mimic-iv-note/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load.sql b/mimic-iv/buildmimic/postgres/load.sql index c2c2f52f..e48848d9 100644 --- a/mimic-iv/buildmimic/postgres/load.sql +++ b/mimic-iv/buildmimic/postgres/load.sql @@ -5,7 +5,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load.sql -- The script assumes the files are in the hosp and icu subfolders of mimic_data_dir -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure correct encoding is defined as -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_7z.sql b/mimic-iv/buildmimic/postgres/load_7z.sql index 8ccc4604..5f238bc4 100644 --- a/mimic-iv/buildmimic/postgres/load_7z.sql +++ b/mimic-iv/buildmimic/postgres/load_7z.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; diff --git a/mimic-iv/buildmimic/postgres/load_gz.sql b/mimic-iv/buildmimic/postgres/load_gz.sql index e9a895a4..f77ac82f 100644 --- a/mimic-iv/buildmimic/postgres/load_gz.sql +++ b/mimic-iv/buildmimic/postgres/load_gz.sql @@ -4,7 +4,7 @@ -- To run from a terminal: -- psql "dbname= user=" -v mimic_data_dir= -f load_gz.sql -\cd :'mimic_data_dir' +\cd :mimic_data_dir -- making sure that all tables are emtpy and correct encoding is defined -utf8- SET CLIENT_ENCODING TO 'utf8'; From 1bb1e339a06bf946388ed41701943208937cac8f Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:45:21 +0300 Subject: [PATCH 27/47] fix: require epi/norepi rate > 0 in pivoted_sofa CV score Same SOFA cardiovascular false-positive as sofa.sql when rates are null. Co-authored-by: Cursor --- mimic-iii/concepts/pivot/pivoted_sofa.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mimic-iii/concepts/pivot/pivoted_sofa.sql b/mimic-iii/concepts/pivot/pivoted_sofa.sql index 6d4940fd..c7947368 100644 --- a/mimic-iii/concepts/pivot/pivoted_sofa.sql +++ b/mimic-iii/concepts/pivot/pivoted_sofa.sql @@ -228,7 +228,7 @@ WITH co AS -- Cardiovascular , cast(case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null From 2d4f3ae0abbd79c24774d1184e6f4c99acb6ed19 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:45:21 +0300 Subject: [PATCH 28/47] fix: keep DuckDB GENERATE_ARRAY as a list for UNNEST Bare GENERATE_SERIES broke icustay_hourly on older DuckDB (#1736). Co-authored-by: Cursor --- .../concepts_duckdb/demographics/icustay_hours.sql | 2 +- .../concepts_duckdb/demographics/icustay_hourly.sql | 2 +- src/mimic_utils/sqlglot_dialects/duckdb.py | 12 ++++++++++++ tests/test_transpile.py | 3 +++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql index 7b459f3e..12624abf 100644 --- a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql +++ b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql @@ -4,7 +4,7 @@ WITH all_hours AS ( SELECT it.icustay_id, STRPTIME(STRFTIME(CAST(it.intime_hr + INTERVAL '59' MINUTE AS TIMESTAMP), '%Y-%m-%d %H:00:00'), '%Y-%m-%d %H:00:00') AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS t(g)) AS hrs FROM mimiciii_derived.icustay_times AS it ) SELECT diff --git a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql index 9341d3a4..682a6f5c 100644 --- a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql +++ b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql @@ -8,7 +8,7 @@ WITH all_hours AS ( THEN it.intime_hr ELSE DATE_TRUNC('HOUR', CAST(it.intime_hr AS TIMESTAMP)) + INTERVAL '1' HOUR END AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS t(g)) AS hrs FROM mimiciv_derived.icustay_times AS it ) SELECT diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 12e83bfd..2c3262b9 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,6 +3,10 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). +- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot +emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a +list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` +reliable across DuckDB versions (#1736). """ import re @@ -41,6 +45,13 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" +# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bc..8e18bad3 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -71,6 +71,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", From a229b37e0944f5cf45f9526e85ed10b7886ea795 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 29/47] fix: include CCO and Arctic Sun temps in vitalsign Itemids 226329/227632/227634 were omitted from temperature (#1358). Co-authored-by: Cursor --- mimic-iv/concepts/measurement/vitalsign.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/measurement/vitalsign.sql b/mimic-iv/concepts/measurement/vitalsign.sql index 030e4055..c2899b4d 100644 --- a/mimic-iv/concepts/measurement/vitalsign.sql +++ b/mimic-iv/concepts/measurement/vitalsign.sql @@ -53,7 +53,7 @@ SELECT AND valuenum < 120 THEN (valuenum - 32) / 1.8 -- already in degC, no conversion necessary - WHEN itemid IN (223762) + WHEN itemid IN (223762, 226329, 227632, 227634) AND valuenum > 10 AND valuenum < 50 THEN valuenum END) @@ -91,7 +91,9 @@ WHERE ce.stay_id IS NOT NULL , 220621 -- Glucose (serum) , 226537 -- Glucose (whole blood) -- TEMPERATURE - -- 226329 -- Blood Temperature CCO (C) + , 226329 -- Blood Temperature CCO (C) + , 227632 -- Arctic Sun/Alsius Temp # 1 C + , 227634 -- Arctic Sun/Alsius Temp # 2 C , 223762 -- "Temperature Celsius" , 223761 -- "Temperature Fahrenheit" , 224642 -- Temperature Site From be93cfab00df02a745cc89384ffdc4d8071232d5 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 30/47] fix: Charlson age_score uses half-open decade bands Age 50 was scored 0; CCI awards 1 point starting at 50. Co-authored-by: Cursor --- mimic-iv/concepts/comorbidity/charlson.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mimic-iv/concepts/comorbidity/charlson.sql b/mimic-iv/concepts/comorbidity/charlson.sql index 2546e7c8..cb6c168c 100644 --- a/mimic-iv/concepts/comorbidity/charlson.sql +++ b/mimic-iv/concepts/comorbidity/charlson.sql @@ -355,10 +355,11 @@ WITH diag AS ( SELECT hadm_id , age - , CASE WHEN age <= 50 THEN 0 - WHEN age <= 60 THEN 1 - WHEN age <= 70 THEN 2 - WHEN age <= 80 THEN 3 + -- CCI age bands are [50,60), [60,70), [70,80), 80+ (not <= decade). + , CASE WHEN age < 50 THEN 0 + WHEN age < 60 THEN 1 + WHEN age < 70 THEN 2 + WHEN age < 80 THEN 3 ELSE 4 END AS age_score FROM `physionet-data.mimiciv_derived.age` ) From aa06aea92db92840baa2c1a4d05770d9012b12bb Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 14:52:09 +0300 Subject: [PATCH 31/47] fix: keep o2-device-only rows after FULL OUTER JOIN WHERE ce.rn = 1 dropped charttimes with devices but no flow. Co-authored-by: Cursor --- mimic-iv/concepts/measurement/oxygen_delivery.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index e76f6303..9c582c36 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -89,8 +89,9 @@ WITH ce_stg1 AS ( FULL OUTER JOIN o2 ON ce.subject_id = o2.subject_id AND ce.charttime = o2.charttime - -- limit to 1 row per subject_id/charttime/itemid from ce_stg2 - WHERE ce.rn = 1 + -- limit to 1 row per subject_id/charttime/itemid from ce_stg2; + -- keep o2-only charttimes (ce side NULL after the full outer join) + WHERE ce.rn = 1 OR ce.subject_id IS NULL ) SELECT From 8f2b9c816ec5c4727b88d56bb95b24a81efbc52e Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:01:28 +0300 Subject: [PATCH 32/47] fix: exclude MetaVision pause gaps from vaso durations Grouping by linkorderid with min/max start/end treated pauses as continuous infusion (#1808). Keep per-row intervals and merge only overlapping ones. Co-authored-by: Cursor --- .../durations/adenosine_durations.sql | 34 +++++++++++++++++-- .../durations/dobutamine_durations.sql | 34 +++++++++++++++++-- .../concepts/durations/dopamine_durations.sql | 34 +++++++++++++++++-- .../durations/epinephrine_durations.sql | 34 +++++++++++++++++-- .../concepts/durations/isuprel_durations.sql | 34 +++++++++++++++++-- .../durations/milrinone_durations.sql | 34 +++++++++++++++++-- .../durations/norepinephrine_durations.sql | 34 +++++++++++++++++-- .../durations/phenylephrine_durations.sql | 34 +++++++++++++++++-- .../durations/vasopressin_durations.sql | 34 +++++++++++++++++-- .../durations/vasopressor_durations.sql | 5 +-- 10 files changed, 282 insertions(+), 29 deletions(-) diff --git a/mimic-iii/concepts/durations/adenosine_durations.sql b/mimic-iii/concepts/durations/adenosine_durations.sql index 4bd2afcd..8859198a 100644 --- a/mimic-iii/concepts/durations/adenosine_durations.sql +++ b/mimic-iii/concepts/durations/adenosine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221282 -- adenosine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dobutamine_durations.sql b/mimic-iii/concepts/durations/dobutamine_durations.sql index bdac0706..7063348c 100644 --- a/mimic-iii/concepts/durations/dobutamine_durations.sql +++ b/mimic-iii/concepts/durations/dobutamine_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221653 -- dobutamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dopamine_durations.sql b/mimic-iii/concepts/durations/dopamine_durations.sql index 65ccea38..1f5672ce 100644 --- a/mimic-iii/concepts/durations/dopamine_durations.sql +++ b/mimic-iii/concepts/durations/dopamine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221662 -- dopamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/epinephrine_durations.sql b/mimic-iii/concepts/durations/epinephrine_durations.sql index a51a4d55..43cfae00 100644 --- a/mimic-iii/concepts/durations/epinephrine_durations.sql +++ b/mimic-iii/concepts/durations/epinephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221289 -- epinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/isuprel_durations.sql b/mimic-iii/concepts/durations/isuprel_durations.sql index b1191881..a468232f 100644 --- a/mimic-iii/concepts/durations/isuprel_durations.sql +++ b/mimic-iii/concepts/durations/isuprel_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 227692 -- Isuprel and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/milrinone_durations.sql b/mimic-iii/concepts/durations/milrinone_durations.sql index 69005edd..4840ba85 100644 --- a/mimic-iii/concepts/durations/milrinone_durations.sql +++ b/mimic-iii/concepts/durations/milrinone_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221986 -- milrinone and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/norepinephrine_durations.sql b/mimic-iii/concepts/durations/norepinephrine_durations.sql index 90b1d949..55317ea1 100644 --- a/mimic-iii/concepts/durations/norepinephrine_durations.sql +++ b/mimic-iii/concepts/durations/norepinephrine_durations.sql @@ -194,11 +194,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221906 -- norepinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -220,6 +248,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/phenylephrine_durations.sql b/mimic-iii/concepts/durations/phenylephrine_durations.sql index 014e364d..c105d358 100644 --- a/mimic-iii/concepts/durations/phenylephrine_durations.sql +++ b/mimic-iii/concepts/durations/phenylephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221749 -- phenylephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressin_durations.sql b/mimic-iii/concepts/durations/vasopressin_durations.sql index 8f93f19f..35b5f688 100644 --- a/mimic-iii/concepts/durations/vasopressin_durations.sql +++ b/mimic-iii/concepts/durations/vasopressin_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 222315 -- vasopressin and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressor_durations.sql b/mimic-iii/concepts/durations/vasopressor_durations.sql index 352c9ae8..64691d8a 100644 --- a/mimic-iii/concepts/durations/vasopressor_durations.sql +++ b/mimic-iii/concepts/durations/vasopressor_durations.sql @@ -263,11 +263,12 @@ ORDER BY s1.icustay_id, s1.starttime -- do not need to group by itemid because we group by linkorderid , vasomv as ( + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + -- Overlapping intervals are merged in vasomv_grp. select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + , starttime, endtime from io_mv - group by icustay_id, linkorderid ) , vasomv_grp as ( From 31c72b6364d238e70bc45db7433e19850b7d0ce0 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:01:28 +0300 Subject: [PATCH 33/47] fix: merge overlapping MetaVision arterial/central line intervals Collapse overlapping procedureevents_mv rows so duration tables no longer emit nested intervals (#371). Co-authored-by: Cursor --- .../durations/arterial_line_durations.sql | 41 ++++++++++++++++--- .../durations/central_line_durations.sql | 41 ++++++++++++++++--- 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/mimic-iii/concepts/durations/arterial_line_durations.sql b/mimic-iii/concepts/durations/arterial_line_durations.sql index 2f87ef30..db11ffa3 100644 --- a/mimic-iii/concepts/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts/durations/arterial_line_durations.sql @@ -163,16 +163,45 @@ with mv as group by icustay_id, arterial_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision arterial line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.arterial_line = 1 + AND t1.arterial_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.arterial_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.arterial_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.arterial_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , arterial_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where arterial_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/central_line_durations.sql b/mimic-iii/concepts/durations/central_line_durations.sql index 3f837c19..2dc9c11d 100644 --- a/mimic-iii/concepts/durations/central_line_durations.sql +++ b/mimic-iii/concepts/durations/central_line_durations.sql @@ -158,16 +158,45 @@ with mv as group by icustay_id, central_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision central line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.central_line = 1 + AND t1.central_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.central_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.central_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.central_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , central_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where central_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; From 9e59a048d4ea477ba88364e4c17c65dca84a4041 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:14:32 +0300 Subject: [PATCH 34/47] fix: Elixhauser no_drg excludes primary and RTRIMs ICD-9 codes WHERE seq_num = 1 contradicted the header (and Quan); padded CHAR codes also failed unpadded BETWEEN ranges (#1077). Co-authored-by: Cursor --- .../concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql index c9f10a78..b3944540 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql @@ -390,8 +390,11 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd -WHERE seq_num = 1 +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` + where seq_num != 1 +) icd ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission From 15c2f7251f16fbd1daef62ef9a24faa345b333f4 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:14:32 +0300 Subject: [PATCH 35/47] fix: AHRQ Elixhauser with DRG uses all diagnoses, RTRIM codes Primary-only filtering skipped secondary codes that DRG exclusions need; RTRIM matches padded ICD-9 storage (#1077). Co-authored-by: Cursor --- mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql index 603a3a4c..245f0eb8 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql @@ -384,8 +384,10 @@ select hadm_id, seq_num, icd9_code when icd9_code = '3091' then 1 when icd9_code = '311' then 1 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd -WHERE seq_num = 1 +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` +) icd ) -- collapse the icd9_code specific flags into hadm_id specific flags -- this groups comorbidities together for a single patient admission From afe7707ecb94c8350f0957157c7158c9c3a96e10 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:15:54 +0300 Subject: [PATCH 36/47] fix: RTRIM ICD-9 codes in Quan Elixhauser Padded codes break exact and prefix matches the same way as AHRQ (#1077). Co-authored-by: Cursor --- mimic-iii/concepts/comorbidity/elixhauser_quan.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql index 18db8c00..7c5a1f2e 100644 --- a/mimic-iii/concepts/comorbidity/elixhauser_quan.sql +++ b/mimic-iii/concepts/comorbidity/elixhauser_quan.sql @@ -174,7 +174,10 @@ select hadm_id, seq_num, icd9_code when SUBSTR(icd9_code, 1, 4) in ('2962','2963','2965','3004') then 1 when SUBSTR(icd9_code, 1, 3) in ('309','311') then 1 else 0 end as depress /* Depression */ -from `physionet-data.mimiciii_clinical.diagnoses_icd` icd +from ( + select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code + from `physionet-data.mimiciii_clinical.diagnoses_icd` +) icd where seq_num != 1 -- we do not include the primary icd-9 code ) -- collapse the icd9_code specific flags into hadm_id specific flags From 70cfd84df1342abd90371546da50f2e5134f5eaf Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 37/47] fix: drop LODS BUN >= 7.50 mg/dL score band That cutpoint is mmol/L; on MIMIC BUN (mg/dL) it scored normal values as 1 (#1065). Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/lods.sql | 3 ++- mimic-iv/concepts/score/lods.sql | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mimic-iii/concepts/severityscores/lods.sql b/mimic-iii/concepts/severityscores/lods.sql index b5c3a7cb..e95d28c8 100644 --- a/mimic-iii/concepts/severityscores/lods.sql +++ b/mimic-iii/concepts/severityscores/lods.sql @@ -174,7 +174,8 @@ select when urineoutput >= 10000.0 then 3 when creatinine_max >= 1.20 then 1 when bun_max >= 17.0 then 1 - when bun_max >= 7.50 then 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) else 0 end as renal diff --git a/mimic-iv/concepts/score/lods.sql b/mimic-iv/concepts/score/lods.sql index bc105402..1bde1f88 100644 --- a/mimic-iv/concepts/score/lods.sql +++ b/mimic-iv/concepts/score/lods.sql @@ -168,7 +168,8 @@ WITH cpap AS ( WHEN urineoutput >= 10000.0 THEN 3 WHEN creatinine_max >= 1.20 THEN 1 WHEN bun_max >= 17.0 THEN 1 - WHEN bun_max >= 7.50 THEN 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) ELSE 0 END AS renal From 5163678441508268f7757fee8750f5420514196a Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 38/47] fix: drop concept tables via bq ls CSV, not cut -f3 Pretty-format column cutting mis-parses table names when rebuilding derived. Co-authored-by: Cursor --- mimic-iv/concepts/make_concepts.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index cc633fc3..42be9d22 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -8,16 +8,15 @@ export MIMIC_VERSION="3.1" # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' -# drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; -do - # skip the first line of dashes - if [[ "${TABLE:0:2}" == '--' ]]; then - continue - fi +# drop existing tables (CSV format: cut -f3 on pretty ls mis-parses names) +while IFS= read -r TABLE; do + [ -z "${TABLE}" ] && continue echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} -done + bq rm -f -q "${TARGET_DATASET}.${TABLE}" +done < <( + bq ls --format=csv --max_results=10000 "physionet-data:${TARGET_DATASET}" \ + | awk -F, 'NR > 1 { print $1 }' +) # create a _version table to store the mimic-iv version, git commit hash, and latest git tag GIT_COMMIT_HASH=$(git rev-parse HEAD) From 6ce72f9cbfbab61ec879c9d80478c8a4af943f80 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:24:42 +0300 Subject: [PATCH 39/47] fix: skip echo charttime parse when time regex misses PARSE_DATETIME on a NULL-concatenated string failed rows without a time. Co-authored-by: Cursor --- mimic-iii/concepts/echo_data.sql | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/mimic-iii/concepts/echo_data.sql b/mimic-iii/concepts/echo_data.sql index 679aa9eb..1582d4ed 100644 --- a/mimic-iii/concepts/echo_data.sql +++ b/mimic-iii/concepts/echo_data.sql @@ -10,13 +10,16 @@ select ROW_ID -- however, the time is available in the echo text, e.g.: -- , substring(ne.text, 'Date/Time: [\[\]0-9*-]+ at ([0-9:]+)') as TIMESTAMP -- we can therefore impute it and re-create charttime - , PARSE_DATETIME - ( - '%Y-%m-%d%H:%M:%S', - FORMAT_DATE('%Y-%m-%d', chartdate) - || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') - || ':00' - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') IS NOT NULL + THEN PARSE_DATETIME( + '%Y-%m-%d%H:%M:%S', + FORMAT_DATE('%Y-%m-%d', chartdate) + || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') + || ':00' + ) + ELSE NULL + END AS charttime -- explanation of below substring: -- 'Indication: ' - matched verbatim From d7ec3d540aecb38f4242f8fddd3ab7902dd69be4 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:00 +0300 Subject: [PATCH 40/47] fix: transpile DATETIME_DIFF MONTH for Postgres MONTH previously KeyError'd in _SECONDS_PER_UNIT; emit year*12+month. Co-authored-by: Cursor --- src/mimic_utils/sqlglot_dialects/postgres.py | 16 ++++++++++++++++ tests/test_transpile.py | 4 ++++ 2 files changed, 20 insertions(+) diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d3369855..509b369b 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 8e18bad3..151d075f 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -219,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction From 1fe4366bbea75a1d6c52fbe2552c03e1a09d0c71 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:00 +0300 Subject: [PATCH 41/47] fix: include CCO/Arctic Sun temps in III vitals Same itemids as IV vitalsign (#1358): 226329, 227632, 227634. Co-authored-by: Cursor --- mimic-iii/concepts/firstday/vitals_first_day.sql | 7 +++++-- mimic-iii/concepts/pivot/pivoted_vital.sql | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mimic-iii/concepts/firstday/vitals_first_day.sql b/mimic-iii/concepts/firstday/vitals_first_day.sql index a77def4d..659278e5 100644 --- a/mimic-iii/concepts/firstday/vitals_first_day.sql +++ b/mimic-iii/concepts/firstday/vitals_first_day.sql @@ -38,7 +38,7 @@ FROM ( when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then 4 -- MeanBP when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then 5 -- RespRate when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then 6 -- TempF, converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then 6 -- TempC + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then 6 -- TempC when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then 7 -- SpO2 when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then 8 -- Glucose @@ -110,7 +110,10 @@ FROM ( 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) pvt diff --git a/mimic-iii/concepts/pivot/pivoted_vital.sql b/mimic-iii/concepts/pivot/pivoted_vital.sql index 27efa0db..12dd7f51 100644 --- a/mimic-iii/concepts/pivot/pivoted_vital.sql +++ b/mimic-iii/concepts/pivot/pivoted_vital.sql @@ -11,7 +11,7 @@ with ce as , (case when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then valuenum else null end) as meanbp , (case when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then valuenum else null end) as resprate , (case when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then (valuenum-32)/1.8 -- converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc , (case when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then valuenum else null end) as spo2 , (case when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then valuenum else null end) as glucose FROM `physionet-data.mimiciii_clinical.chartevents` ce @@ -74,7 +74,10 @@ with ce as 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) From 0d80d2b5880a0eb2fdb3e43fa94c409b1a49cc3c Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:35:01 +0300 Subject: [PATCH 42/47] fix: include CCO/Arctic Sun temps in aline_vitals Co-authored-by: Cursor --- mimic-iii/notebooks/aline/aline_vitals.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iii/notebooks/aline/aline_vitals.sql b/mimic-iii/notebooks/aline/aline_vitals.sql index 9ee60b79..033be2c6 100644 --- a/mimic-iii/notebooks/aline/aline_vitals.sql +++ b/mimic-iii/notebooks/aline/aline_vitals.sql @@ -10,13 +10,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -32,6 +32,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) From fc005949d211e120861aa0755982ee31c39eb666 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 43/47] fix: skip echo height PARSE when time regex misses Co-authored-by: Cursor --- mimic-iii/concepts/pivot/pivoted_height.sql | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mimic-iii/concepts/pivot/pivoted_height.sql b/mimic-iii/concepts/pivot/pivoted_height.sql index 61452739..b481b0aa 100644 --- a/mimic-iii/concepts/pivot/pivoted_height.sql +++ b/mimic-iii/concepts/pivot/pivoted_height.sql @@ -83,12 +83,16 @@ WITH ht_in AS subject_id -- extract the time of the note from the text itself -- add this to the structured date in the chartdate column - , PARSE_DATETIME('%b-%d-%Y%H:%M', - CONCAT( - FORMAT_DATE("%b-%d-%Y", chartdate), - REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') - ) - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') IS NOT NULL + THEN PARSE_DATETIME('%b-%d-%Y%H:%M', + CONCAT( + FORMAT_DATE("%b-%d-%Y", chartdate), + REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') + ) + ) + ELSE NULL + END AS charttime -- sometimes numeric values contain de-id numbers, e.g. [** Numeric Identifier **] -- this case is used to ignore that text , case From b836c5a242781d9af305de464636fcdbeea109ad Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 44/47] fix: include CCO/Arctic Sun temps in aline-aws vitals Co-authored-by: Cursor --- mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql index 6d30869c..7aadd4da 100644 --- a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql @@ -9,13 +9,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -31,6 +31,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) From 1cab27c6446ccbfb3f28e9ef0e455294c7dbcdb9 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:46:08 +0300 Subject: [PATCH 45/47] fix: convert SAPS glucose/BUN thresholds to mg/dL SAPS 1984 publishes mmol/L cutpoints; MIMIC labs are mg/dL. Apply PhysioNet Challenge 2012 conversions (glucose x18, BUN x2.8). Co-authored-by: Cursor --- mimic-iii/concepts/severityscores/saps.sql | 33 ++++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/mimic-iii/concepts/severityscores/saps.sql b/mimic-iii/concepts/severityscores/saps.sql index ec961f5d..f4d01eba 100644 --- a/mimic-iii/concepts/severityscores/saps.sql +++ b/mimic-iii/concepts/severityscores/saps.sql @@ -195,13 +195,15 @@ select , case when bun_max is null then null - when bun_max >= 55.0 then 4 - when bun_max >= 36.0 then 3 - when bun_max >= 29.0 then 2 - when bun_max >= 7.50 then 1 - when bun_min < 3.5 then 1 - when bun_max >= 3.5 and bun_max < 7.5 - and bun_min >= 3.5 and bun_min < 7.5 + -- SAPS 1984 publishes urea in mmol/L; MIMIC BUN is mg/dL (×2.8). + -- Same conversion as PhysioNet Challenge 2012 saps_score.m. + when bun_max >= 154.0 then 4 + when bun_max >= 100.8 then 3 + when bun_max >= 81.2 then 2 + when bun_max >= 21.0 then 1 + when bun_min < 9.8 then 1 + when bun_max >= 9.8 and bun_max < 21.0 + and bun_min >= 9.8 and bun_min < 21.0 then 0 end as bun_score @@ -231,14 +233,15 @@ select , case when glucose_max is null then null - when glucose_max >= 44.5 then 4 - when glucose_min < 1.6 then 4 - when glucose_max >= 27.8 then 3 - when glucose_min < 2.8 then 3 - when glucose_min < 3.9 then 2 - when glucose_max >= 14.0 then 1 - when glucose_max >= 3.9 and glucose_max < 14.0 - and glucose_min >= 3.9 and glucose_min < 14.0 + -- SAPS 1984 glucose is mmol/L; MIMIC glucose is mg/dL (×18). + when glucose_max >= 801.0 then 4 + when glucose_min < 28.8 then 4 + when glucose_max >= 500.4 then 3 + when glucose_min < 50.4 then 3 + when glucose_min < 70.2 then 2 + when glucose_max >= 252.0 then 1 + when glucose_max >= 70.2 and glucose_max < 252.0 + and glucose_min >= 70.2 and glucose_min < 252.0 then 0 end as glucose_score From cb5f9ac811679ece99dca4b25df6e3c3f42fea28 Mon Sep 17 00:00:00 2001 From: Taksh Date: Thu, 23 Jul 2026 15:55:15 +0300 Subject: [PATCH 46/47] fix: build icustay_times/hours before pivoted_oasis pivoted_oasis reads mimiciii_derived.icustay_hours; make-concepts never created it (or its icustay_times dependency). Co-authored-by: Cursor --- mimic-iii/concepts/make-concepts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index 01e15f00..7e74e3b8 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -22,12 +22,17 @@ echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.heightweight" < "demographics/heightweight.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_detail" < "demographics/icustay_detail.sql" +# icustay_hours (and pivoted_oasis) need icustay_times first +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_times" < "demographics/icustay_times.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_hours" < "demographics/icustay_hours.sql" # Durations (usually of treatments) echo 'Directory 2: durations' bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_classification" < "durations/ventilation_classification.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_durations" < "durations/ventilation_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.crrt_durations" < "durations/crrt_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.arterial_line_durations" < "durations/arterial_line_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.central_line_durations" < "durations/central_line_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.adenosine_durations" < "durations/adenosine_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_durations" < "durations/dobutamine_durations.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_durations" < "durations/dopamine_durations.sql" @@ -46,6 +51,7 @@ bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_dose" < bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_dose" < "durations/norepinephrine_dose.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_dose" < "durations/phenylephrine_dose.sql" bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_dose" < "durations/vasopressin_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.neuroblock_dose" < "durations/neuroblock_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' From ca766fc8aa9c8d6de8c2af062cbed62bb7357e24 Mon Sep 17 00:00:00 2001 From: Taksh Date: Fri, 24 Jul 2026 09:25:58 +0300 Subject: [PATCH 47/47] fix: include pivoted_oasis in postgres/duckdb concept builds SQL existed but make scripts never ran it (#830). Co-authored-by: Cursor --- mimic-iii/concepts_duckdb/duckdb.sql | 2 ++ mimic-iii/concepts_postgres/postgres-make-concepts.sql | 1 + 2 files changed, 3 insertions(+) diff --git a/mimic-iii/concepts_duckdb/duckdb.sql b/mimic-iii/concepts_duckdb/duckdb.sql index dc49fd86..2635b2ad 100644 --- a/mimic-iii/concepts_duckdb/duckdb.sql +++ b/mimic-iii/concepts_duckdb/duckdb.sql @@ -93,6 +93,8 @@ .read pivot/pivoted_vital.sql .print 'pivot/pivoted_bg_art.sql' .read pivot/pivoted_bg_art.sql +.print 'pivot/pivoted_oasis.sql' +.read pivot/pivoted_oasis.sql .print 'pivot/pivoted_sofa.sql' .read pivot/pivoted_sofa.sql diff --git a/mimic-iii/concepts_postgres/postgres-make-concepts.sql b/mimic-iii/concepts_postgres/postgres-make-concepts.sql index 3779bef6..318b3da1 100644 --- a/mimic-iii/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iii/concepts_postgres/postgres-make-concepts.sql @@ -61,6 +61,7 @@ SET search_path TO mimiciii_derived, mimiciii; \i pivot/pivoted_uo.sql \i pivot/pivoted_vital.sql \i pivot/pivoted_bg_art.sql +\i pivot/pivoted_oasis.sql \i pivot/pivoted_sofa.sql -- comorbidity