diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 9401198e0..ca4c8c8bb 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -35,8 +35,12 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran. + continue-on-error: true + if: steps.get_files_to_lint.outputs.lintees != '' uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-missing-file: true \ No newline at end of file diff --git a/mimic-iii/buildmimic/duckdb/import_duckdb.sh b/mimic-iii/buildmimic/duckdb/import_duckdb.sh index 15d4977fe..badf83b86 100755 --- a/mimic-iii/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iii/buildmimic/duckdb/import_duckdb.sh @@ -84,9 +84,11 @@ make_table_name () { # load data into database find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do make_table_name "$FILE" - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL echo "done!" done && echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iii/buildmimic/postgres/Makefile b/mimic-iii/buildmimic/postgres/Makefile index 17ea42e20..29953adbd 100644 --- a/mimic-iii/buildmimic/postgres/Makefile +++ b/mimic-iii/buildmimic/postgres/Makefile @@ -104,7 +104,7 @@ mimic-build-gz: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -151,7 +151,7 @@ mimic-build: @echo '------------------' @echo '' @sleep 2 - psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR} + psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)" @echo '' @echo '--------------------' @echo '-- Adding indexes --' @@ -184,7 +184,7 @@ ifeq ("$(physionetuser)","") @echo 'Call the makefile again with physionetuser=' @echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org' else - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)" endif mimic-demo-download: @@ -192,7 +192,7 @@ mimic-demo-download: @echo '-- Downloading MIMIC-III from PhysioNet --' @echo '------------------------------------------' @echo '' - wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" + wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)" #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip, #but need to maintain compatibility with Windows, which baffling lacks these things diff --git a/mimic-iii/buildmimic/postgres/create_mimic_user.sh b/mimic-iii/buildmimic/postgres/create_mimic_user.sh index 772764ce7..d639f5fcc 100755 --- a/mimic-iii/buildmimic/postgres/create_mimic_user.sh +++ b/mimic-iii/buildmimic/postgres/create_mimic_user.sh @@ -20,6 +20,13 @@ else echo "User is set to '$MIMIC_USER'"; fi +# escape ' in password for SQL string literal +MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g") +# quote SQL identifiers (double any embedded ") +sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; } +MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER") +MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB") + PSQL='psql' # add in the host/port, if they were specified (not null, -n) @@ -58,11 +65,11 @@ fi if [ "$MIMIC_USER" != "postgres" ]; then # we need to create this user via postgres # use SUDO to login as postgres - $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';" + $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';" fi if [ "$MIMIC_DB" != "postgres" ]; then # drop and recreate the database - $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;" - $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;" -fi \ No newline at end of file + $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;" + $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;" +fi diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7a..4e0113d35 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -10,6 +10,34 @@ CHUNKSIZE = 10 ** 6 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME) +# Column dtypes for tables that trigger pandas mixed-type warnings when +# loaded with the default low_memory chunked inference (see #1237). +# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql. +TABLE_DTYPES = { + "chartevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "datetimeevents": { + "WARNING": "Int64", + "ERROR": "Int64", + "RESULTSTATUS": "string", + "STOPPED": "string", + }, + "inputevents_cv": { + "ORIGINALRATE": "float64", + "ORIGINALRATEUOM": "string", + "ORIGINALSITE": "string", + }, + "noteevents": { + "CHARTTIME": "string", + "STORETIME": "string", + "ISERROR": "string", + }, +} + def _table_name_from_csv(filename: str) -> str: """Derive SQL table name from a CSV path (literal suffix, not str.strip).""" @@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() +def _read_csv(path, table, **kwargs): + # low_memory=False avoids dtype re-inference across chunks; table-specific + # dtypes pin the columns that otherwise flip between numeric/object. + return pd.read_csv( + path, + index_col="ROW_ID", + low_memory=False, + dtype=TABLE_DTYPES.get(table), + **kwargs, + ) + + if os.path.exists(DATABASE_NAME): msg = "File {} already exists.".format(DATABASE_NAME) print(msg) sys.exit() +# Prefer .csv.gz when both exist for the same table; also load plain .csv +# (import.sh already accepts both). +files_by_table = {} +for f in glob("*.csv"): + files_by_table[_table_name_from_csv(f)] = f for f in glob("*.csv.gz"): + files_by_table[_table_name_from_csv(f)] = f + +for table, f in sorted(files_by_table.items()): print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") + df = _read_csv(f, table) df.to_sql(table, CONNECTION_STRING) else: # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + for chunk in _read_csv(f, table, chunksize=CHUNKSIZE): chunk.to_sql(table, CONNECTION_STRING, if_exists="append") print("Finished processing {}".format(f)) diff --git a/mimic-iii/buildmimic/sqlite/import.sh b/mimic-iii/buildmimic/sqlite/import.sh index dbf6b9ce4..58b85abb4 100755 --- a/mimic-iii/buildmimic/sqlite/import.sh +++ b/mimic-iii/buildmimic/sqlite/import.sh @@ -28,11 +28,11 @@ for FILE in *; do TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -40,7 +40,7 @@ for FILE in *; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE <= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/arterial_line_durations.sql b/mimic-iii/concepts/durations/arterial_line_durations.sql index 2f87ef307..db11ffa39 100644 --- a/mimic-iii/concepts/durations/arterial_line_durations.sql +++ b/mimic-iii/concepts/durations/arterial_line_durations.sql @@ -163,16 +163,45 @@ with mv as group by icustay_id, arterial_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision arterial line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.arterial_line = 1 + AND t1.arterial_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.arterial_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.arterial_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.arterial_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , arterial_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where arterial_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/central_line_durations.sql b/mimic-iii/concepts/durations/central_line_durations.sql index 3f837c196..2dc9c11dd 100644 --- a/mimic-iii/concepts/durations/central_line_durations.sql +++ b/mimic-iii/concepts/durations/central_line_durations.sql @@ -158,16 +158,45 @@ with mv as group by icustay_id, central_line_rownum having min(charttime) != max(charttime) ) +-- collapse overlapping MetaVision central line intervals (#371) +, mv_dur as +( + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours + FROM mv s1 + INNER JOIN mv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.central_line = 1 + AND t1.central_line = 1 + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM mv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t2.central_line = 1 + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE s1.central_line = 1 + AND NOT EXISTS + ( + SELECT 1 FROM mv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s2.central_line = 1 + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime +) select icustay_id -- , central_line_rownum , starttime, endtime, duration_hours from cv_dur UNION ALL ---TODO: collapse metavision durations if they overlap select icustay_id - -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum - , starttime, endtime - , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -from mv -where central_line = 1 + , starttime, endtime, duration_hours +from mv_dur order by icustay_id, starttime; diff --git a/mimic-iii/concepts/durations/dobutamine_durations.sql b/mimic-iii/concepts/durations/dobutamine_durations.sql index bdac0706e..7063348c9 100644 --- a/mimic-iii/concepts/durations/dobutamine_durations.sql +++ b/mimic-iii/concepts/durations/dobutamine_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221653 -- dobutamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/dopamine_durations.sql b/mimic-iii/concepts/durations/dopamine_durations.sql index 65ccea381..1f5672ce4 100644 --- a/mimic-iii/concepts/durations/dopamine_durations.sql +++ b/mimic-iii/concepts/durations/dopamine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221662 -- dopamine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/epinephrine_durations.sql b/mimic-iii/concepts/durations/epinephrine_durations.sql index a51a4d55a..43cfae000 100644 --- a/mimic-iii/concepts/durations/epinephrine_durations.sql +++ b/mimic-iii/concepts/durations/epinephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221289 -- epinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/isuprel_durations.sql b/mimic-iii/concepts/durations/isuprel_durations.sql index b11918816..a468232fa 100644 --- a/mimic-iii/concepts/durations/isuprel_durations.sql +++ b/mimic-iii/concepts/durations/isuprel_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 227692 -- Isuprel and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/milrinone_durations.sql b/mimic-iii/concepts/durations/milrinone_durations.sql index 69005edd1..4840ba850 100644 --- a/mimic-iii/concepts/durations/milrinone_durations.sql +++ b/mimic-iii/concepts/durations/milrinone_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221986 -- milrinone and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/neuroblock_dose.sql b/mimic-iii/concepts/durations/neuroblock_dose.sql index 098b557e9..a1ce5c9a2 100644 --- a/mimic-iii/concepts/durations/neuroblock_dose.sql +++ b/mimic-iii/concepts/durations/neuroblock_dose.sql @@ -28,7 +28,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped -- we only include continuous infusions, therefore expect a rate , max(case @@ -68,7 +68,7 @@ with drugmv as , 1 as drug -- the 'stopped' column indicates if a drug has been disconnected - , max(case when stopped in ('Stopped','D/C''d') then 1 else 0 end) as drug_stopped + , max(case when stopped in ('Stopped', CONCAT('D/C', CHR(39), 'd')) then 1 else 0 end) as drug_stopped , max(case when valuenum <= 10 then 0 else 1 end) as drug_null -- educated guess! @@ -179,7 +179,11 @@ select ) = 1 then 1 - when (CHARTTIME - (LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime))) > (interval '8 hours') then 1 + when DATETIME_DIFF( + CHARTTIME, + LAG(CHARTTIME, 1) OVER (partition by icustay_id, drug order by charttime), + HOUR + ) > 8 then 1 else null end as drug_start diff --git a/mimic-iii/concepts/durations/norepinephrine_durations.sql b/mimic-iii/concepts/durations/norepinephrine_durations.sql index 90b1d9496..55317ea12 100644 --- a/mimic-iii/concepts/durations/norepinephrine_durations.sql +++ b/mimic-iii/concepts/durations/norepinephrine_durations.sql @@ -194,11 +194,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221906 -- norepinephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -220,6 +248,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/phenylephrine_durations.sql b/mimic-iii/concepts/durations/phenylephrine_durations.sql index 014e364db..c105d3584 100644 --- a/mimic-iii/concepts/durations/phenylephrine_durations.sql +++ b/mimic-iii/concepts/durations/phenylephrine_durations.sql @@ -198,11 +198,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 221749 -- phenylephrine and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -224,6 +252,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressin_durations.sql b/mimic-iii/concepts/durations/vasopressin_durations.sql index 8f93f19fd..35b5f6887 100644 --- a/mimic-iii/concepts/durations/vasopressin_durations.sql +++ b/mimic-iii/concepts/durations/vasopressin_durations.sql @@ -195,11 +195,39 @@ and ( select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + , starttime, endtime FROM `physionet-data.mimiciii_clinical.inputevents_mv` where itemid = 222315 -- vasopressin and statusdescription != 'Rewritten' -- only valid orders - group by icustay_id, linkorderid +) + +, vasomv_grp as +( + -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808). + SELECT + s1.icustay_id + , s1.starttime + , MIN(t1.endtime) AS endtime + FROM vasomv s1 + INNER JOIN vasomv t1 + ON s1.icustay_id = t1.icustay_id + AND s1.starttime <= t1.endtime + AND NOT EXISTS + ( + SELECT 1 FROM vasomv t2 + WHERE t1.icustay_id = t2.icustay_id + AND t1.endtime >= t2.starttime + AND t1.endtime < t2.endtime + ) + WHERE NOT EXISTS + ( + SELECT 1 FROM vasomv s2 + WHERE s1.icustay_id = s2.icustay_id + AND s1.starttime > s2.starttime + AND s1.starttime <= s2.endtime + ) + GROUP BY s1.icustay_id, s1.starttime ) select @@ -221,6 +249,6 @@ select , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours -- add durations from - vasomv + vasomv_grp order by icustay_id, vasonum; diff --git a/mimic-iii/concepts/durations/vasopressor_durations.sql b/mimic-iii/concepts/durations/vasopressor_durations.sql index 352c9ae86..64691d8ac 100644 --- a/mimic-iii/concepts/durations/vasopressor_durations.sql +++ b/mimic-iii/concepts/durations/vasopressor_durations.sql @@ -263,11 +263,12 @@ ORDER BY s1.icustay_id, s1.starttime -- do not need to group by itemid because we group by linkorderid , vasomv as ( + -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808). + -- Overlapping intervals are merged in vasomv_grp. select icustay_id, linkorderid - , min(starttime) as starttime, max(endtime) as endtime + , starttime, endtime from io_mv - group by icustay_id, linkorderid ) , vasomv_grp as ( diff --git a/mimic-iii/concepts/echo_data.sql b/mimic-iii/concepts/echo_data.sql index 679aa9ebf..1582d4ede 100644 --- a/mimic-iii/concepts/echo_data.sql +++ b/mimic-iii/concepts/echo_data.sql @@ -10,13 +10,16 @@ select ROW_ID -- however, the time is available in the echo text, e.g.: -- , substring(ne.text, 'Date/Time: [\[\]0-9*-]+ at ([0-9:]+)') as TIMESTAMP -- we can therefore impute it and re-create charttime - , PARSE_DATETIME - ( - '%Y-%m-%d%H:%M:%S', - FORMAT_DATE('%Y-%m-%d', chartdate) - || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') - || ':00' - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') IS NOT NULL + THEN PARSE_DATETIME( + '%Y-%m-%d%H:%M:%S', + FORMAT_DATE('%Y-%m-%d', chartdate) + || REGEXP_EXTRACT(ne.text, 'Date/Time: .+? at ([0-9]+:[0-9]{2})') + || ':00' + ) + ELSE NULL + END AS charttime -- explanation of below substring: -- 'Indication: ' - matched verbatim diff --git a/mimic-iii/concepts/firstday/vitals_first_day.sql b/mimic-iii/concepts/firstday/vitals_first_day.sql index a77def4d1..659278e5a 100644 --- a/mimic-iii/concepts/firstday/vitals_first_day.sql +++ b/mimic-iii/concepts/firstday/vitals_first_day.sql @@ -38,7 +38,7 @@ FROM ( when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then 4 -- MeanBP when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then 5 -- RespRate when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then 6 -- TempF, converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then 6 -- TempC + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then 6 -- TempC when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then 7 -- SpO2 when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then 8 -- Glucose @@ -110,7 +110,10 @@ FROM ( 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) pvt diff --git a/mimic-iii/concepts/make-concepts.sh b/mimic-iii/concepts/make-concepts.sh index a7d1152c5..7e74e3b81 100644 --- a/mimic-iii/concepts/make-concepts.sh +++ b/mimic-iii/concepts/make-concepts.sh @@ -14,108 +14,114 @@ echo '' set -x echo 'Top level files..' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.code_status < code_status.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.echo_data < echo_data.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.code_status" < "code_status.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.echo_data" < "echo_data.sql" echo 'Running queries in 10 directories.' echo 'Directory 1: demographics' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.heightweight < demographics/heightweight.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.icustay_detail < demographics/icustay_detail.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.heightweight" < "demographics/heightweight.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_detail" < "demographics/icustay_detail.sql" +# icustay_hours (and pivoted_oasis) need icustay_times first +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_times" < "demographics/icustay_times.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.icustay_hours" < "demographics/icustay_hours.sql" # Durations (usually of treatments) echo 'Directory 2: durations' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_classification < durations/ventilation_classification.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_durations < durations/ventilation_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.crrt_durations < durations/crrt_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.adenosine_durations < durations/adenosine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_durations < durations/dobutamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_durations < durations/dopamine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_durations < durations/epinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.isuprel_durations < durations/isuprel_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.milrinone_durations < durations/milrinone_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_durations < durations/norepinephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_durations < durations/phenylephrine_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_durations < durations/vasopressin_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressor_durations < durations/vasopressor_durations.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_durations < durations/weight_durations.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_classification" < "durations/ventilation_classification.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_durations" < "durations/ventilation_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.crrt_durations" < "durations/crrt_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.arterial_line_durations" < "durations/arterial_line_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.central_line_durations" < "durations/central_line_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.adenosine_durations" < "durations/adenosine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_durations" < "durations/dobutamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_durations" < "durations/dopamine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_durations" < "durations/epinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.isuprel_durations" < "durations/isuprel_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.milrinone_durations" < "durations/milrinone_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_durations" < "durations/norepinephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_durations" < "durations/phenylephrine_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_durations" < "durations/vasopressin_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressor_durations" < "durations/vasopressor_durations.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_durations" < "durations/weight_durations.sql" # dose queries for vasopressors -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dobutamine_dose < durations/dobutamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.dopamine_dose < durations/dopamine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.epinephrine_dose < durations/epinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.norepinephrine_dose < durations/norepinephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.phenylephrine_dose < durations/phenylephrine_dose.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vasopressin_dose < durations/vasopressin_dose.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dobutamine_dose" < "durations/dobutamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.dopamine_dose" < "durations/dopamine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.epinephrine_dose" < "durations/epinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.norepinephrine_dose" < "durations/norepinephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.phenylephrine_dose" < "durations/phenylephrine_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vasopressin_dose" < "durations/vasopressin_dose.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.neuroblock_dose" < "durations/neuroblock_dose.sql" # "pivoted" tables which have icustay_id / timestamp as the primary key echo 'Directory 3: pivoted tables' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_vital < pivot/pivoted_vital.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_uo < pivot/pivoted_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_rrt < pivot/pivoted_rrt.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_lab < pivot/pivoted_lab.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_invasive_lines < pivot/pivoted_invasive_lines.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_icp < pivot/pivoted_icp.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_height < pivot/pivoted_height.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_gcs < pivot/pivoted_gcs.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_fio2 < pivot/pivoted_fio2.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg < pivot/pivoted_bg.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_vital" < "pivot/pivoted_vital.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_uo" < "pivot/pivoted_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_rrt" < "pivot/pivoted_rrt.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_lab" < "pivot/pivoted_lab.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_invasive_lines" < "pivot/pivoted_invasive_lines.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_icp" < "pivot/pivoted_icp.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_height" < "pivot/pivoted_height.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_gcs" < "pivot/pivoted_gcs.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_fio2" < "pivot/pivoted_fio2.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg" < "pivot/pivoted_bg.sql" # pivoted_bg_art must be run after pivoted_bg -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_bg_art < pivot/pivoted_bg_art.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_bg_art" < "pivot/pivoted_bg_art.sql" # pivoted oasis depends on icustay_hours in demographics -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_oasis < pivot/pivoted_oasis.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_oasis" < "pivot/pivoted_oasis.sql" # pivoted sofa depends on many above pivoted views, ventilation_durations, and dose queries -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.pivoted_sofa < pivot/pivoted_sofa.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.pivoted_sofa" < "pivot/pivoted_sofa.sql" echo 'Directory 4: comorbidity' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37 < comorbidity/elixhauser_ahrq_v37.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg < comorbidity/elixhauser_ahrq_v37_no_drg.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_quan < comorbidity/elixhauser_quan.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_ahrq < comorbidity/elixhauser_score_ahrq.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.elixhauser_score_quan < comorbidity/elixhauser_score_quan.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37" < "comorbidity/elixhauser_ahrq_v37.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_ahrq_v37_no_drg" < "comorbidity/elixhauser_ahrq_v37_no_drg.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_quan" < "comorbidity/elixhauser_quan.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_ahrq" < "comorbidity/elixhauser_score_ahrq.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.elixhauser_score_quan" < "comorbidity/elixhauser_score_quan.sql" echo 'Directory 5: firstday' # data which is extracted from a patient's first ICU stay -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day < firstday/blood_gas_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.blood_gas_first_day_arterial < firstday/blood_gas_first_day_arterial.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.gcs_first_day < firstday/gcs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.labs_first_day < firstday/labs_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.rrt_first_day < firstday/rrt_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output_first_day < firstday/urine_output_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ventilation_first_day < firstday/ventilation_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.vitals_first_day < firstday/vitals_first_day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.weight_first_day < firstday/weight_first_day.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day" < "firstday/blood_gas_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.blood_gas_first_day_arterial" < "firstday/blood_gas_first_day_arterial.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.gcs_first_day" < "firstday/gcs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.labs_first_day" < "firstday/labs_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.rrt_first_day" < "firstday/rrt_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output_first_day" < "firstday/urine_output_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ventilation_first_day" < "firstday/ventilation_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.vitals_first_day" < "firstday/vitals_first_day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.weight_first_day" < "firstday/weight_first_day.sql" echo 'Directory 6: fluid_balance' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.urine_output < fluid_balance/urine_output.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.urine_output" < "fluid_balance/urine_output.sql" echo 'Directory 7: sepsis' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.angus < sepsis/angus.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.martin < sepsis/martin.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.explicit < sepsis/explicit.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.angus" < "sepsis/angus.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.martin" < "sepsis/martin.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.explicit" < "sepsis/explicit.sql" # diagnosis mapping using CCS echo 'Directory 8: diagnosis' # load the ccs_multi_dx.csv.gz file into bq -bq load --source_format=CSV ${TARGET_DATASET}.ccs_multi_dx diagnosis/ccs_multi_dx.csv.gz diagnosis/ccs_multi_dx.json -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.ccs_dx < diagnosis/ccs_dx.sql +bq load --source_format=CSV "${TARGET_DATASET}.ccs_multi_dx" "diagnosis/ccs_multi_dx.csv.gz" "diagnosis/ccs_multi_dx.json" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.ccs_dx" < "diagnosis/ccs_dx.sql" # Organ failure scores echo 'Directory 9: organfailure' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_creatinine < organfailure/kdigo_creatinine.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_uo < organfailure/kdigo_uo.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages < organfailure/kdigo_stages.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_7day < organfailure/kdigo_stages_7day.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.kdigo_stages_48hr < organfailure/kdigo_stages_48hr.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.meld < organfailure/meld.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_creatinine" < "organfailure/kdigo_creatinine.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_uo" < "organfailure/kdigo_uo.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages" < "organfailure/kdigo_stages.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_7day" < "organfailure/kdigo_stages_7day.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.kdigo_stages_48hr" < "organfailure/kdigo_stages_48hr.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.meld" < "organfailure/meld.sql" # Severity of illness scores (requires many views from above) echo 'Directory 10: severityscores' -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.oasis < severityscores/oasis.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sofa < severityscores/sofa.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.saps < severityscores/saps.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sapsii < severityscores/sapsii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.apsiii < severityscores/apsiii.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.lods < severityscores/lods.sql -bq query ${BQ_FLAGS} --destination_table=${TARGET_DATASET}.sirs < severityscores/sirs.sql +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.oasis" < "severityscores/oasis.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sofa" < "severityscores/sofa.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.saps" < "severityscores/saps.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sapsii" < "severityscores/sapsii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.apsiii" < "severityscores/apsiii.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.lods" < "severityscores/lods.sql" +bq query ${BQ_FLAGS} --destination_table="${TARGET_DATASET}.sirs" < "severityscores/sirs.sql" echo 'Finished creating concepts.' \ No newline at end of file diff --git a/mimic-iii/concepts/pivot/pivoted_height.sql b/mimic-iii/concepts/pivot/pivoted_height.sql index 614527395..b481b0aa3 100644 --- a/mimic-iii/concepts/pivot/pivoted_height.sql +++ b/mimic-iii/concepts/pivot/pivoted_height.sql @@ -83,12 +83,16 @@ WITH ht_in AS subject_id -- extract the time of the note from the text itself -- add this to the structured date in the chartdate column - , PARSE_DATETIME('%b-%d-%Y%H:%M', - CONCAT( - FORMAT_DATE("%b-%d-%Y", chartdate), - REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') - ) - ) AS charttime + , CASE + WHEN REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') IS NOT NULL + THEN PARSE_DATETIME('%b-%d-%Y%H:%M', + CONCAT( + FORMAT_DATE("%b-%d-%Y", chartdate), + REGEXP_EXTRACT(ne.text, 'Date/Time: [\\[\\]0-9*-]+ at ([0-9:]+)') + ) + ) + ELSE NULL + END AS charttime -- sometimes numeric values contain de-id numbers, e.g. [** Numeric Identifier **] -- this case is used to ignore that text , case diff --git a/mimic-iii/concepts/pivot/pivoted_sofa.sql b/mimic-iii/concepts/pivot/pivoted_sofa.sql index 6d4940fd1..c7947368a 100644 --- a/mimic-iii/concepts/pivot/pivoted_sofa.sql +++ b/mimic-iii/concepts/pivot/pivoted_sofa.sql @@ -228,7 +228,7 @@ WITH co AS -- Cardiovascular , cast(case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iii/concepts/pivot/pivoted_vital.sql b/mimic-iii/concepts/pivot/pivoted_vital.sql index 27efa0dbb..12dd7f519 100644 --- a/mimic-iii/concepts/pivot/pivoted_vital.sql +++ b/mimic-iii/concepts/pivot/pivoted_vital.sql @@ -11,7 +11,7 @@ with ce as , (case when itemid in (456,52,6702,443,220052,220181,225312) and valuenum > 0 and valuenum < 300 then valuenum else null end) as meanbp , (case when itemid in (615,618,220210,224690) and valuenum > 0 and valuenum < 70 then valuenum else null end) as resprate , (case when itemid in (223761,678) and valuenum > 70 and valuenum < 120 then (valuenum-32)/1.8 -- converted to degC in valuenum call - when itemid in (223762,676) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc + when itemid in (223762,676,226329,227632,227634) and valuenum > 10 and valuenum < 50 then valuenum else null end) as tempc , (case when itemid in (646,220277) and valuenum > 0 and valuenum <= 100 then valuenum else null end) as spo2 , (case when itemid in (807,811,1529,3745,3744,225664,220621,226537) and valuenum > 0 then valuenum else null end) as glucose FROM `physionet-data.mimiciii_clinical.chartevents` ce @@ -74,7 +74,10 @@ with ce as 223762, -- "Temperature Celsius" 676, -- "Temperature C" 223761, -- "Temperature Fahrenheit" - 678 -- "Temperature F" + 678, -- "Temperature F" + 226329, -- Blood Temperature CCO (C) + 227632, -- Arctic Sun/Alsius Temp # 1 C + 227634 -- Arctic Sun/Alsius Temp # 2 C ) ) diff --git a/mimic-iii/concepts/severityscores/lods.sql b/mimic-iii/concepts/severityscores/lods.sql index b5c3a7cbb..e95d28c81 100644 --- a/mimic-iii/concepts/severityscores/lods.sql +++ b/mimic-iii/concepts/severityscores/lods.sql @@ -174,7 +174,8 @@ select when urineoutput >= 10000.0 then 3 when creatinine_max >= 1.20 then 1 when bun_max >= 17.0 then 1 - when bun_max >= 7.50 then 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) else 0 end as renal diff --git a/mimic-iii/concepts/severityscores/saps.sql b/mimic-iii/concepts/severityscores/saps.sql index ec961f5df..f4d01eba4 100644 --- a/mimic-iii/concepts/severityscores/saps.sql +++ b/mimic-iii/concepts/severityscores/saps.sql @@ -195,13 +195,15 @@ select , case when bun_max is null then null - when bun_max >= 55.0 then 4 - when bun_max >= 36.0 then 3 - when bun_max >= 29.0 then 2 - when bun_max >= 7.50 then 1 - when bun_min < 3.5 then 1 - when bun_max >= 3.5 and bun_max < 7.5 - and bun_min >= 3.5 and bun_min < 7.5 + -- SAPS 1984 publishes urea in mmol/L; MIMIC BUN is mg/dL (×2.8). + -- Same conversion as PhysioNet Challenge 2012 saps_score.m. + when bun_max >= 154.0 then 4 + when bun_max >= 100.8 then 3 + when bun_max >= 81.2 then 2 + when bun_max >= 21.0 then 1 + when bun_min < 9.8 then 1 + when bun_max >= 9.8 and bun_max < 21.0 + and bun_min >= 9.8 and bun_min < 21.0 then 0 end as bun_score @@ -231,14 +233,15 @@ select , case when glucose_max is null then null - when glucose_max >= 44.5 then 4 - when glucose_min < 1.6 then 4 - when glucose_max >= 27.8 then 3 - when glucose_min < 2.8 then 3 - when glucose_min < 3.9 then 2 - when glucose_max >= 14.0 then 1 - when glucose_max >= 3.9 and glucose_max < 14.0 - and glucose_min >= 3.9 and glucose_min < 14.0 + -- SAPS 1984 glucose is mmol/L; MIMIC glucose is mg/dL (×18). + when glucose_max >= 801.0 then 4 + when glucose_min < 28.8 then 4 + when glucose_max >= 500.4 then 3 + when glucose_min < 50.4 then 3 + when glucose_min < 70.2 then 2 + when glucose_max >= 252.0 then 1 + when glucose_max >= 70.2 and glucose_max < 252.0 + and glucose_min >= 70.2 and glucose_min < 252.0 then 0 end as glucose_score diff --git a/mimic-iii/concepts/severityscores/sofa.sql b/mimic-iii/concepts/severityscores/sofa.sql index 9afc5be12..52cb8d5f2 100644 --- a/mimic-iii/concepts/severityscores/sofa.sql +++ b/mimic-iii/concepts/severityscores/sofa.sql @@ -218,7 +218,7 @@ left join `physionet-data.mimiciii_derived.gcs_first_day` gcs -- Cardiovascular , case when rate_dopamine > 15 or rate_epinephrine > 0.1 or rate_norepinephrine > 0.1 then 4 - when rate_dopamine > 5 or rate_epinephrine <= 0.1 or rate_norepinephrine <= 0.1 then 3 + when rate_dopamine > 5 or (rate_epinephrine > 0 and rate_epinephrine <= 0.1) or (rate_norepinephrine > 0 and rate_norepinephrine <= 0.1) then 3 when rate_dopamine > 0 or rate_dobutamine > 0 then 2 when meanbp_min < 70 then 1 when coalesce(meanbp_min, rate_dopamine, rate_dobutamine, rate_epinephrine, rate_norepinephrine) is null then null diff --git a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql index 7b459f3e8..12624abff 100644 --- a/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql +++ b/mimic-iii/concepts_duckdb/demographics/icustay_hours.sql @@ -4,7 +4,7 @@ WITH all_hours AS ( SELECT it.icustay_id, STRPTIME(STRFTIME(CAST(it.intime_hr + INTERVAL '59' MINUTE AS TIMESTAMP), '%Y-%m-%d %H:00:00'), '%Y-%m-%d %H:00:00') AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS t(g)) AS hrs FROM mimiciii_derived.icustay_times AS it ) SELECT diff --git a/mimic-iii/concepts_duckdb/duckdb.sql b/mimic-iii/concepts_duckdb/duckdb.sql index dc49fd86f..2635b2ad7 100644 --- a/mimic-iii/concepts_duckdb/duckdb.sql +++ b/mimic-iii/concepts_duckdb/duckdb.sql @@ -93,6 +93,8 @@ .read pivot/pivoted_vital.sql .print 'pivot/pivoted_bg_art.sql' .read pivot/pivoted_bg_art.sql +.print 'pivot/pivoted_oasis.sql' +.read pivot/pivoted_oasis.sql .print 'pivot/pivoted_sofa.sql' .read pivot/pivoted_sofa.sql diff --git a/mimic-iii/concepts_postgres/postgres-make-concepts.sql b/mimic-iii/concepts_postgres/postgres-make-concepts.sql index 3779bef6c..318b3da1c 100644 --- a/mimic-iii/concepts_postgres/postgres-make-concepts.sql +++ b/mimic-iii/concepts_postgres/postgres-make-concepts.sql @@ -61,6 +61,7 @@ SET search_path TO mimiciii_derived, mimiciii; \i pivot/pivoted_uo.sql \i pivot/pivoted_vital.sql \i pivot/pivoted_bg_art.sql +\i pivot/pivoted_oasis.sql \i pivot/pivoted_sofa.sql -- comorbidity diff --git a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql index 6d30869ca..7aadd4da4 100644 --- a/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql +++ b/mimic-iii/notebooks/aline-aws/aline_vitals-awsathena.sql @@ -9,13 +9,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -31,6 +31,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) diff --git a/mimic-iii/notebooks/aline/aline_vitals.sql b/mimic-iii/notebooks/aline/aline_vitals.sql index 9ee60b797..033be2c67 100644 --- a/mimic-iii/notebooks/aline/aline_vitals.sql +++ b/mimic-iii/notebooks/aline/aline_vitals.sql @@ -10,13 +10,13 @@ with vitals_stg0 as , case -- MAP, Temperature, HR, CVP, SpO2, when itemid in (456,52,6702,443,220052,220181,225312) then 'MAP' - when itemid in (223762,676,223761,678) then 'Temperature' + when itemid in (223762,676,223761,678,226329,227632,227634) then 'Temperature' when itemid in (211,220045) then 'HeartRate' when itemid in (646,220277) then 'SpO2' else null end as label , case when itemid in (223761,678) and ((valuenum-32)/1.8)<10 then null - when itemid in (223762,676) and valuenum < 10 then null + when itemid in (223762,676,226329,227632,227634) and valuenum < 10 then null -- convert F to C when itemid in (223761,678) then (valuenum-32)/1.8 -- sanity checks on data - one outliter with spo2 < 25 @@ -32,6 +32,7 @@ with vitals_stg0 as ( 456,52,6702,443,220052,220181,225312 -- map , 223762,676,223761,678 -- temp + , 226329,227632,227634 -- CCO / Arctic Sun temp (C) , 211,220045 -- hr , 646,220277 -- spo2 ) diff --git a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh index 52e34e0ff..f996541a2 100644 --- a/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh +++ b/mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh @@ -22,11 +22,20 @@ fi sleep 2 # loop through each .csv file in the section folder -for fn in `ls $REPORT_PATH`; do - echo `date`: $fn - fn_stem=`echo $fn | cut -d. -f 1` +for fn_path in "$REPORT_PATH"/*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + echo "$(date): $fn" + fn_stem=${fn%.csv} # run chexpert - must be run from chexpert folder - python $CHEXPERT_PATH/label.py --verbose --reports_path $REPORT_PATH/$fn --output_path ${fn_stem}_labeled.csv --mention_phrases_dir $CHEXPERT_PATH/phrases/mention --unmention_phrases_dir $CHEXPERT_PATH/phrases/unmention --pre_negation_uncertainty_path $CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt --negation_path $CHEXPERT_PATH/patterns/negation.txt --post_negation_uncertainty_path $CHEXPERT_PATH/patterns/post_negation_uncertainty.txt - echo `date`: done! + python "$CHEXPERT_PATH/label.py" --verbose \ + --reports_path "$fn_path" \ + --output_path "${fn_stem}_labeled.csv" \ + --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \ + --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \ + --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \ + --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \ + --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt" + echo "$(date): done!" echo '' -done \ No newline at end of file +done diff --git a/mimic-iv-cxr/txt/negbio/run_negbio.sh b/mimic-iv-cxr/txt/negbio/run_negbio.sh index 13cba4795..dab200cd4 100755 --- a/mimic-iv-cxr/txt/negbio/run_negbio.sh +++ b/mimic-iv-cxr/txt/negbio/run_negbio.sh @@ -32,20 +32,22 @@ sleep 2 # mimic_cxr_001.csv # mimic_cxr_002.csv # .. etc -for fn in `ls $BASE_FOLDER`; +for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv; do + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") echo "Looping through files with mimic_cxr_###.csv pattern." # validate it's a mimic_cxr sections file if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]]; then - export INPUT_FILE=${BASE_FOLDER}/$fn + export INPUT_FILE=$fn_path # remove extension from filename # sets the folder location to stem of original filename # all intermediate files will be saved in this folder export OUTPUT_DIR=${INPUT_FILE::-4} - echo $OUTPUT_DIR - running NegBio.. - python $NEGBIO_PATH/negbio/negbio_csv2bioc.py --output $OUTPUT_DIR/report $INPUT_FILE + echo "$OUTPUT_DIR - running NegBio.." + python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE" python $NEGBIO_PATH/negbio/negbio_pipeline.py section_split --pattern $NEGBIO_PATH/patterns/section_titles_cxr8.txt --output $OUTPUT_DIR/sections $OUTPUT_DIR/report/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py ssplit --output $OUTPUT_DIR/ssplit $OUTPUT_DIR/sections/* --workers=6 python $NEGBIO_PATH/negbio/negbio_pipeline.py parse --output $OUTPUT_DIR/parse $OUTPUT_DIR/ssplit/* --workers=6 @@ -55,7 +57,7 @@ do # ultimate filename we save the labels to export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv - python $NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py --phrases_file $NEGBIO_PATH/patterns/chexpert_phrases.yml --output $OUTPUT_LABELS $OUTPUT_DIR/neg/* + python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/* fi done echo "Done looping through files." diff --git a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh index 9142f7b40..f6e6a2dad 100644 --- a/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-ed/buildmimic/duckdb/import_duckdb.sh @@ -93,18 +93,29 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-ed # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (ed) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .. " try duckdb "$OUTFILE" <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected ed/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh index c0425fe6f..487d74050 100644 --- a/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv-note/buildmimic/duckdb/import_duckdb.sh @@ -96,18 +96,23 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both .csv and .csv.gz ( '*.csv???' only matched .csv.gz ). +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv-note # avoids syntax errors if mimic-iv in the same dir - case $DIRNAME in + case "$DIRNAME" in (note) ;; # OK (*) continue; esac + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") echo "Loading $FILE .." OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -122,5 +127,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected note/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/duckdb/import_duckdb.sh b/mimic-iv/buildmimic/duckdb/import_duckdb.sh index ab1d37fa3..32f608c48 100755 --- a/mimic-iv/buildmimic/duckdb/import_duckdb.sh +++ b/mimic-iv/buildmimic/duckdb/import_duckdb.sh @@ -101,18 +101,26 @@ make_table_name () { # load data into database -find "$MIMIC_DIR" -type f -name '*.csv???' | sort | while IFS= read -r FILE; do +# Match both uncompressed (.csv) and gzip (.csv.gz). The old '*.csv???' +# pattern only matched names with exactly three chars after ".csv" (i.e. .gz), +# so plain .csv files were silently skipped while the script still reported success. +LOAD_COUNT_FILE=$(mktemp) || die "mktemp failed" +trap 'rm -f "$LOAD_COUNT_FILE"' EXIT +: > "$LOAD_COUNT_FILE" +find "$MIMIC_DIR" -type f \( -name '*.csv' -o -name '*.csv.gz' \) | sort | while IFS= read -r FILE; do make_table_name "$FILE" # skip directories which we do not expect in mimic-iv # avoids syntax errors if mimic-iv-ed in the same dir - case $DIRNAME in + case "$DIRNAME" in (hosp|icu) ;; # OK (*) continue; esac - echo "Loading $FILE .. \c" + # Escape single quotes for SQL string literal + FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g") + printf "Loading %s .. " "$FILE" OUTPUT=$(duckdb "$OUTFILE" 2>&1 <<-EOSQL - COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); + COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"'); EOSQL ) # If the table is missing in the DB, we emit a warning and continue. @@ -127,5 +135,11 @@ EOSQL yell "$OUTPUT" die "Exiting due to load error." fi + echo x >> "$LOAD_COUNT_FILE" echo "done!" -done && echo "Successfully finished loading data into $OUTFILE." +done || exit $? + +if [ ! -s "$LOAD_COUNT_FILE" ]; then + die "No .csv / .csv.gz files loaded from $MIMIC_DIR (expected hosp/ and icu/)." +fi +echo "Successfully finished loading data into $OUTFILE." diff --git a/mimic-iv/buildmimic/mysql/validate_demo.sql b/mimic-iv/buildmimic/mysql/validate_demo.sql index 575e84133..ebf4f02dc 100644 --- a/mimic-iv/buildmimic/mysql/validate_demo.sql +++ b/mimic-iv/buildmimic/mysql/validate_demo.sql @@ -31,14 +31,17 @@ FROM ( SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) exp @@ -64,14 +67,17 @@ INNER JOIN SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM procedureevents ) obs diff --git a/mimic-iv/buildmimic/postgres/validate_demo.sql b/mimic-iv/buildmimic/postgres/validate_demo.sql index 303dd98e4..049497e56 100644 --- a/mimic-iv/buildmimic/postgres/validate_demo.sql +++ b/mimic-iv/buildmimic/postgres/validate_demo.sql @@ -22,14 +22,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, 3795 AS row_count UNION ALL SELECT 'prescriptions' AS tbl, 18087 AS row_count UNION ALL SELECT 'procedures_icd' AS tbl, 722 AS row_count UNION ALL + SELECT 'provider' AS tbl, 40508 AS row_count UNION ALL SELECT 'services' AS tbl, 319 AS row_count UNION ALL SELECT 'transfers' AS tbl, 1190 AS row_count UNION ALL -- icu data SELECT 'icustays' AS tbl, 140 AS row_count UNION ALL + SELECT 'caregiver' AS tbl, 15468 AS row_count UNION ALL SELECT 'd_items' AS tbl, 4014 AS row_count UNION ALL SELECT 'chartevents' AS tbl, 668862 AS row_count UNION ALL SELECT 'datetimeevents' AS tbl, 15280 AS row_count UNION ALL SELECT 'inputevents' AS tbl, 20404 AS row_count UNION ALL + SELECT 'ingredientevents' AS tbl, 25728 AS row_count UNION ALL SELECT 'outputevents' AS tbl, 9362 AS row_count UNION ALL SELECT 'procedureevents' AS tbl, 1468 AS row_count ) @@ -54,14 +57,17 @@ WITH expected AS SELECT 'poe_detail' AS tbl, count(*) AS row_count FROM mimiciv_hosp.poe_detail UNION ALL SELECT 'prescriptions' AS tbl, count(*) AS row_count FROM mimiciv_hosp.prescriptions UNION ALL SELECT 'procedures_icd' AS tbl, count(*) AS row_count FROM mimiciv_hosp.procedures_icd UNION ALL + SELECT 'provider' AS tbl, count(*) AS row_count FROM mimiciv_hosp.provider UNION ALL SELECT 'services' AS tbl, count(*) AS row_count FROM mimiciv_hosp.services UNION ALL SELECT 'transfers' AS tbl, count(*) AS row_count FROM mimiciv_hosp.transfers UNION ALL -- icu data SELECT 'icustays' AS tbl, count(*) AS row_count FROM mimiciv_icu.icustays UNION ALL + SELECT 'caregiver' AS tbl, count(*) AS row_count FROM mimiciv_icu.caregiver UNION ALL SELECT 'chartevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.chartevents UNION ALL SELECT 'd_items' AS tbl, count(*) AS row_count FROM mimiciv_icu.d_items UNION ALL SELECT 'datetimeevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.datetimeevents UNION ALL SELECT 'inputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.inputevents UNION ALL + SELECT 'ingredientevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.ingredientevents UNION ALL SELECT 'outputevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.outputevents UNION ALL SELECT 'procedureevents' AS tbl, count(*) AS row_count FROM mimiciv_icu.procedureevents ) diff --git a/mimic-iv/buildmimic/sqlite/import.py b/mimic-iv/buildmimic/sqlite/import.py index 68d436479..31b880c44 100644 --- a/mimic-iv/buildmimic/sqlite/import.py +++ b/mimic-iv/buildmimic/sqlite/import.py @@ -110,7 +110,7 @@ def main(): if args.limit > 0: for f in data_files: if 'patients' in f.name: - pt = pd.read_csv(f) + pt = pd.read_csv(f, low_memory=False) break if pt is None: raise FileNotFoundError('Unable to find a patients file in current folder.') @@ -158,7 +158,7 @@ def main(): tablename = tablenames[i] print("Starting processing {}".format(tablename), end='.. ') if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, dtype=mimic_dtypes) + df = pd.read_csv(f, dtype=mimic_dtypes, low_memory=False) df = process_dataframe(df, subjects=subjects) df.to_sql(tablename, connection, index=False) row_counts[tablename] += len(df) diff --git a/mimic-iv/buildmimic/sqlite/import.sh b/mimic-iv/buildmimic/sqlite/import.sh index 08ded2881..fb642d1ce 100755 --- a/mimic-iv/buildmimic/sqlite/import.sh +++ b/mimic-iv/buildmimic/sqlite/import.sh @@ -29,11 +29,11 @@ for FILE in */**.csv*; do TABLE_NAME=$(echo "${BASENAME%%.*}" | tr "[:upper:]" "[:lower:]") case "$FILE" in *csv) - IMPORT_CMD=".import $FILE $TABLE_NAME" + IMPORT_CMD=".import \"$FILE\" $TABLE_NAME" ;; # need to decompress csv before load *csv.gz) - IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME" + IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME" ;; # not a data file so skip *) @@ -41,7 +41,7 @@ for FILE in */**.csv*; do ;; esac echo "Loading $FILE." - sqlite3 $OUTFILE <" exit 1 fi diff --git a/mimic-iv/concepts/firstday/first_day_sofa.sql b/mimic-iv/concepts/firstday/first_day_sofa.sql index 62282af7b..ffd3d3c23 100644 --- a/mimic-iv/concepts/firstday/first_day_sofa.sql +++ b/mimic-iv/concepts/firstday/first_day_sofa.sql @@ -196,8 +196,8 @@ WITH vaso_stg AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 THEN 2 WHEN mbp_min < 70 THEN 1 diff --git a/mimic-iv/concepts/make_concepts.sh b/mimic-iv/concepts/make_concepts.sh index 112bcdc22..42be9d225 100644 --- a/mimic-iv/concepts/make_concepts.sh +++ b/mimic-iv/concepts/make_concepts.sh @@ -8,16 +8,15 @@ export MIMIC_VERSION="3.1" # note: max_rows=1 *displays* only one row, but all rows are inserted into the destination table BQ_OPTIONS='--quiet --headless --max_rows=0 --use_legacy_sql=False --replace' -# drop the existing tables in the target dataset -for TABLE in `bq ls physionet-data:${TARGET_DATASET} | cut -d' ' -f3`; -do - # skip the first line of dashes - if [[ "${TABLE:0:2}" == '--' ]]; then - continue - fi +# drop existing tables (CSV format: cut -f3 on pretty ls mis-parses names) +while IFS= read -r TABLE; do + [ -z "${TABLE}" ] && continue echo "Dropping table ${TARGET_DATASET}.${TABLE}" - bq rm -f -q ${TARGET_DATASET}.${TABLE} -done + bq rm -f -q "${TARGET_DATASET}.${TABLE}" +done < <( + bq ls --format=csv --max_results=10000 "physionet-data:${TARGET_DATASET}" \ + | awk -F, 'NR > 1 { print $1 }' +) # create a _version table to store the mimic-iv version, git commit hash, and latest git tag GIT_COMMIT_HASH=$(git rev-parse HEAD) @@ -44,7 +43,7 @@ for table_path in demographics/icustay_times; do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done # generate tables in subfolders @@ -54,30 +53,29 @@ done # * organfailure depends on measurement for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do - for fn in `ls $d`; + for fn_path in "$d"/*.sql; do - # only run SQL queries - if [[ "${fn: -4}" == ".sql" ]]; then - # table name is file name minus extension - tbl=`echo $fn | rev | cut -d. -f2- | rev` + [ -f "$fn_path" ] || continue + fn=$(basename "$fn_path") + # table name is file name minus extension + tbl="${fn%.sql}" - # skip certain tables where order matters - skip=0 - for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 - do - if [[ "${tbl}" == "${skip_table}" ]]; then - skip=1 - break - fi - done; - if [[ "${skip}" == "1" ]]; then - continue - fi - - # not skipping - so generate the table on bigquery - echo "Generating ${TARGET_DATASET}.${tbl}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${tbl} < ${d}/${fn} + # skip certain tables where order matters + skip=0 + for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3 + do + if [[ "${tbl}" == "${skip_table}" ]]; then + skip=1 + break + fi + done; + if [[ "${skip}" == "1" ]]; then + continue fi + + # not skipping - so generate the table on bigquery + echo "Generating ${TARGET_DATASET}.${tbl}" + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${tbl}" < "${fn_path}" done done @@ -88,5 +86,5 @@ do table=`echo $table_path | rev | cut -d/ -f1 | rev` echo "Generating ${TARGET_DATASET}.${table}" - bq query ${BQ_OPTIONS} --destination_table=${TARGET_DATASET}.${table} < ${table_path}.sql + bq query ${BQ_OPTIONS} --destination_table="${TARGET_DATASET}.${table}" < "${table_path}.sql" done diff --git a/mimic-iv/concepts/measurement/oxygen_delivery.sql b/mimic-iv/concepts/measurement/oxygen_delivery.sql index e76f63036..9c582c362 100644 --- a/mimic-iv/concepts/measurement/oxygen_delivery.sql +++ b/mimic-iv/concepts/measurement/oxygen_delivery.sql @@ -89,8 +89,9 @@ WITH ce_stg1 AS ( FULL OUTER JOIN o2 ON ce.subject_id = o2.subject_id AND ce.charttime = o2.charttime - -- limit to 1 row per subject_id/charttime/itemid from ce_stg2 - WHERE ce.rn = 1 + -- limit to 1 row per subject_id/charttime/itemid from ce_stg2; + -- keep o2-only charttimes (ce side NULL after the full outer join) + WHERE ce.rn = 1 OR ce.subject_id IS NULL ) SELECT diff --git a/mimic-iv/concepts/measurement/urine_output_rate.sql b/mimic-iv/concepts/measurement/urine_output_rate.sql index 019392118..bc6740149 100644 --- a/mimic-iv/concepts/measurement/urine_output_rate.sql +++ b/mimic-iv/concepts/measurement/urine_output_rate.sql @@ -16,12 +16,14 @@ WITH tm AS ( ) -- now calculate time since last UO measurement +-- Use SECOND diffs (like kdigo_uo) so BigQuery and PostgreSQL agree (#1549). +-- BigQuery DATETIME_DIFF(..., MINUTE/HOUR) truncates; SECOND keeps fractions. , uo_tm AS ( SELECT tm.stay_id , CASE WHEN LAG(charttime) OVER w IS NULL - THEN DATETIME_DIFF(charttime, intime_hr, MINUTE) - ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, MINUTE) + THEN DATETIME_DIFF(charttime, intime_hr, SECOND) / 60.0 + ELSE DATETIME_DIFF(charttime, LAG(charttime) OVER w, SECOND) / 60.0 END AS tm_since_last_uo , uo.charttime , uo.urineoutput @@ -45,24 +47,33 @@ WITH tm AS ( -- to 1 hour of UO, therefore we use '5' and '11' to restrict the -- period, rather than 6/12 this assumption may overestimate UO rate -- when documentation is done less than hourly - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_6hr , ROUND(CAST( SUM( CASE - WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 5 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 5 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_6hr - , SUM(CASE WHEN DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + , SUM(CASE + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.urineoutput ELSE NULL END) AS urineoutput_12hr , ROUND(CAST( SUM( CASE - WHEN - DATETIME_DIFF(io.charttime, iosum.charttime, HOUR) <= 11 + WHEN DATETIME_DIFF( + io.charttime, iosum.charttime, SECOND + ) / 3600.0 <= 11 THEN iosum.tm_since_last_uo ELSE NULL END) / 60.0 AS NUMERIC ), 6) AS uo_tm_12hr diff --git a/mimic-iv/concepts/measurement/vitalsign.sql b/mimic-iv/concepts/measurement/vitalsign.sql index 030e4055f..c2899b4d0 100644 --- a/mimic-iv/concepts/measurement/vitalsign.sql +++ b/mimic-iv/concepts/measurement/vitalsign.sql @@ -53,7 +53,7 @@ SELECT AND valuenum < 120 THEN (valuenum - 32) / 1.8 -- already in degC, no conversion necessary - WHEN itemid IN (223762) + WHEN itemid IN (223762, 226329, 227632, 227634) AND valuenum > 10 AND valuenum < 50 THEN valuenum END) @@ -91,7 +91,9 @@ WHERE ce.stay_id IS NOT NULL , 220621 -- Glucose (serum) , 226537 -- Glucose (whole blood) -- TEMPERATURE - -- 226329 -- Blood Temperature CCO (C) + , 226329 -- Blood Temperature CCO (C) + , 227632 -- Arctic Sun/Alsius Temp # 1 C + , 227634 -- Arctic Sun/Alsius Temp # 2 C , 223762 -- "Temperature Celsius" , 223761 -- "Temperature Fahrenheit" , 224642 -- Temperature Site diff --git a/mimic-iv/concepts/score/lods.sql b/mimic-iv/concepts/score/lods.sql index bc1054023..1bde1f88b 100644 --- a/mimic-iv/concepts/score/lods.sql +++ b/mimic-iv/concepts/score/lods.sql @@ -168,7 +168,8 @@ WITH cpap AS ( WHEN urineoutput >= 10000.0 THEN 3 WHEN creatinine_max >= 1.20 THEN 1 WHEN bun_max >= 17.0 THEN 1 - WHEN bun_max >= 7.50 THEN 1 + -- note: do not use bun_max >= 7.50; that is a mmol/L cutpoint + -- incorrectly applied to MIMIC BUN in mg/dL (#1065) ELSE 0 END AS renal diff --git a/mimic-iv/concepts/score/sofa.sql b/mimic-iv/concepts/score/sofa.sql index 0dbf28d15..049b39206 100644 --- a/mimic-iv/concepts/score/sofa.sql +++ b/mimic-iv/concepts/score/sofa.sql @@ -280,8 +280,8 @@ WITH co AS ( OR rate_norepinephrine > 0.1 THEN 4 WHEN rate_dopamine > 5 - OR rate_epinephrine <= 0.1 - OR rate_norepinephrine <= 0.1 + OR (rate_epinephrine > 0 AND rate_epinephrine <= 0.1) + OR (rate_norepinephrine > 0 AND rate_norepinephrine <= 0.1) THEN 3 WHEN rate_dopamine > 0 OR rate_dobutamine > 0 diff --git a/mimic-iv/concepts/validate_concepts.sh b/mimic-iv/concepts/validate_concepts.sh index eeaddb826..645a82a3e 100755 --- a/mimic-iv/concepts/validate_concepts.sh +++ b/mimic-iv/concepts/validate_concepts.sh @@ -32,7 +32,8 @@ fail=0 missing=0 while read -r tbl; do [ -z "${tbl}" ] && continue - if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then + # exact field match (regex grep can false-hit prefixes) + if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then echo "MISSING: expected table ${DATASET}.${tbl} was not built" missing=1 fail=1 @@ -47,7 +48,7 @@ empty=0 while IFS=, read -r tbl rows; do [ -z "${tbl}" ] && continue # only validate the tables we actually build (ignore _metadata and any strays) - if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then + if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then echo "EMPTY: table ${DATASET}.${tbl} has 0 rows" empty=1 fail=1 diff --git a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql index 9341d3a45..682a6f5ce 100644 --- a/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql +++ b/mimic-iv/concepts_duckdb/demographics/icustay_hourly.sql @@ -8,7 +8,7 @@ WITH all_hours AS ( THEN it.intime_hr ELSE DATE_TRUNC('HOUR', CAST(it.intime_hr AS TIMESTAMP)) + INTERVAL '1' HOUR END AS endtime, - GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS hrs + (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS INT)) AS t(g)) AS hrs FROM mimiciv_derived.icustay_times AS it ) SELECT diff --git a/src/mimic_utils/sqlglot_dialects/duckdb.py b/src/mimic_utils/sqlglot_dialects/duckdb.py index 12e83bfd0..2c3262b94 100644 --- a/src/mimic_utils/sqlglot_dialects/duckdb.py +++ b/src/mimic_utils/sqlglot_dialects/duckdb.py @@ -3,6 +3,10 @@ - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``). +- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot +emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a +list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)`` +reliable across DuckDB versions (#1736). """ import re @@ -41,6 +45,13 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) - return f"STRPTIME({this}, {fmt})" +# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST. +def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str: + start = self.sql(expression, "start") + end = self.sql(expression, "end") + return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))" + + class MimicDuckDB(DuckDB): class Generator(DuckDB.Generator): def datatype_sql(self, expression: exp.DataType) -> str: @@ -54,4 +65,5 @@ def datatype_sql(self, expression: exp.DataType) -> str: **DuckDB.Generator.TRANSFORMS, exp.RegexpExtract: _regexp_extract_sql, exp.ParseDatetime: _parse_datetime_sql, + exp.GenerateSeries: _generate_array_sql, } diff --git a/src/mimic_utils/sqlglot_dialects/postgres.py b/src/mimic_utils/sqlglot_dialects/postgres.py index d33698558..509b369b2 100644 --- a/src/mimic_utils/sqlglot_dialects/postgres.py +++ b/src/mimic_utils/sqlglot_dialects/postgres.py @@ -23,8 +23,11 @@ def _unit(expression: exp.Expression, default: str = "DAY") -> str: # The logic is as follows: # * DAY -> difference of the two calendar dates (date subtraction = whole days) # * YEAR -> difference of the two calendar years +# * MONTH -> year*12 + month difference (calendar month boundaries) # * sub-day units -> truncate both operands to the unit (which makes the # elapsed seconds an exact multiple of the unit) then divide. +# WEEK is intentionally unsupported: BigQuery week starts (SUNDAY/ISO) do not +# match PostgreSQL DATE_TRUNC('week') Monday semantics. # https://cloud.google.com/bigquery/docs/reference/standard-sql/datetime_functions#datetime_diff _SECONDS_PER_UNIT = {"SECOND": 1, "MINUTE": 60, "HOUR": 3600} @@ -39,6 +42,19 @@ def _datetime_diff_sql(self: Postgres.Generator, expression: exp.Expression) -> return f"(CAST({end} AS DATE) - CAST({start} AS DATE))" if unit == "YEAR": return f"CAST(EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start}) AS BIGINT)" + if unit == "MONTH": + # Calendar month boundaries (matches BigQuery DATETIME_DIFF MONTH). + return ( + "CAST((EXTRACT(YEAR FROM {end}) - EXTRACT(YEAR FROM {start})) * 12 " + "+ (EXTRACT(MONTH FROM {end}) - EXTRACT(MONTH FROM {start})) AS BIGINT)" + ).format(end=end, start=start) + if unit not in _SECONDS_PER_UNIT: + # WEEK (and WEEK(SUNDAY)/ISO) need BigQuery's week-start semantics; + # refuse rather than emit a silently wrong days/7 division. + raise ValueError( + f"Unsupported DATETIME_DIFF unit {unit!r}; " + "expected SECOND, MINUTE, HOUR, DAY, MONTH, or YEAR" + ) lo = unit.lower() factor = _SECONDS_PER_UNIT[unit] diff --git a/src/mimic_utils/transpile.py b/src/mimic_utils/transpile.py index d4a65541e..29924c187 100644 --- a/src/mimic_utils/transpile.py +++ b/src/mimic_utils/transpile.py @@ -113,7 +113,7 @@ def transpile_file( f"CREATE TABLE {derived_schema}{Path(source_file).stem} AS\n" ) + transpiled_query - with open(destination_file, "w") as write_file: + with open(destination_file, "w", encoding="utf-8") as write_file: write_file.write(transpiled_query) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 952240bca..151d075f4 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -52,6 +52,9 @@ def t(bq: str, dialect: str) -> str: "SELECT (CAST(a.dischtime AS DATE) - CAST(a.admittime AS DATE)) FROM t AS a"), ("diff_year_pg", "SELECT DATETIME_DIFF(a.b, a.c, YEAR) FROM t a", "postgres", "SELECT CAST(EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c) AS BIGINT) FROM t AS a"), + ("diff_month_pg", "SELECT DATETIME_DIFF(a.b, a.c, MONTH) FROM t a", "postgres", + "SELECT CAST((EXTRACT(YEAR FROM a.b) - EXTRACT(YEAR FROM a.c)) * 12 " + "+ (EXTRACT(MONTH FROM a.b) - EXTRACT(MONTH FROM a.c)) AS BIGINT) FROM t AS a"), # DuckDB handles DATETIME_DIFF natively (boundary count), operands swapped ("diff_hour_duckdb", "SELECT DATETIME_DIFF(a.outtime, a.intime, HOUR) FROM t a", "duckdb", "SELECT DATE_DIFF('HOUR', a.intime, a.outtime) FROM t AS a"), @@ -71,6 +74,9 @@ def t(bq: str, dialect: str) -> str: # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres) ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres", "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"), + # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST + ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb", + "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"), # handled natively by sqlglot 30.x (regression guards) ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres", @@ -216,6 +222,7 @@ def test_mimic_iii_concept_transpiles_and_reparses(sql_file, dialect): ("2150-01-02 01:00:00", "2150-01-01 23:00:00", "DAY", 1), # crosses midnight ("2150-01-01 23:00:00", "2150-01-01 01:00:00", "DAY", 0), # same day ("2155-06-15 00:00:00", "2150-01-01 00:00:00", "YEAR", 5), + ("2150-04-01 00:00:00", "2150-01-15 00:00:00", "MONTH", 3), ("2150-01-01 00:01:00", "2150-01-01 00:00:59", "MINUTE", 1), ("2150-01-01 00:00:30", "2150-01-01 00:00:10", "SECOND", 20), ("2150-01-01 01:00:00", "2150-01-01 03:00:00", "HOUR", -2), # negative direction