fix: UO rate, vaso pause gaps, line durations, load paths

.github/workflows/lint_sqlfluff.yml

@@ -35,8 +35,12 @@ jobs:
         shell: bash
         run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
       - name: Annotate
+        # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran.
+        continue-on-error: true
+        if: steps.get_files_to_lint.outputs.lintees != ''
         uses: yuzutech/annotations-action@v0.6.0
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           title: "SQLFluff Lint"
-          input: "./annotations.json"
+          input: "./annotations.json"
+          ignore-missing-file: true

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -84,9 +84,11 @@ make_table_name () {
 # load data into database
 find "$MIMIC_DIR" -type f -regex '.*\.csv\(.gz\)*' | while IFS= read -r FILE; do
     make_table_name "$FILE"
-    echo "Loading $FILE .. \c"
+    # Escape single quotes for SQL string literal
+    FILE_SQL=$(printf '%s' "$FILE" | sed "s/'/''/g")
+    printf "Loading %s .. " "$FILE"
     try duckdb "$OUTFILE" <<-EOSQL
-		COPY $TABLE_NAME FROM '$FILE' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
+		COPY $TABLE_NAME FROM '$FILE_SQL' (HEADER, DELIM ',', QUOTE '"', ESCAPE '"');
 EOSQL
     echo "done!"
 done && echo "Successfully finished loading data into $OUTFILE."

mimic-iii/buildmimic/postgres/Makefile

@@ -104,7 +104,7 @@ mimic-build-gz:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v mimic_data_dir=${datadir}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data_gz.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -151,7 +151,7 @@ mimic-build:
 	@echo '------------------'
 	@echo ''
 	@sleep 2
-	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v mimic_data_dir=${DATADIR}
+	psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f postgres_load_data.sql -v "mimic_data_dir=$(DATADIR)"
 	@echo ''
 	@echo '--------------------'
 	@echo '-- Adding indexes --'
@@ -184,15 +184,15 @@ ifeq ("$(physionetuser)","")
 	@echo 'Call the makefile again with physionetuser=<USERNAME>'
 	@echo ' e.g. make eicu-download datadir=/path/to/data physionetuser=hello@physionet.org'
 else
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETURL)"
 endif
 
 mimic-demo-download:
 	@echo '------------------------------------------'
 	@echo '-- Downloading MIMIC-III from PhysioNet --'
 	@echo '------------------------------------------'
 	@echo ''
-	wget --user $(physionetuser) --ask-password -P $(DATADIR) -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
+	wget --user $(physionetuser) --ask-password -P "$(DATADIR)" -A csv.gz -m -p -E -k -K -np -nd "$(PHYSIONETDEMOURL)"
 
 #This is fairly inelegant and could be tidied with a for loop and an if to check for gzip,
 #but need to maintain compatibility with Windows, which baffling lacks these things

mimic-iii/buildmimic/postgres/create_mimic_user.sh

@@ -20,6 +20,13 @@ else
   echo "User is set to '$MIMIC_USER'";
 fi
 
+# escape ' in password for SQL string literal
+MIMIC_PASSWORD_SQL=$(printf '%s' "$MIMIC_PASSWORD" | sed "s/'/''/g")
+# quote SQL identifiers (double any embedded ")
+sql_ident () { printf '%s' "$1" | sed 's/"/""/g; s/^/"/; s/$/"/'; }
+MIMIC_USER_SQL=$(sql_ident "$MIMIC_USER")
+MIMIC_DB_SQL=$(sql_ident "$MIMIC_DB")
+
 PSQL='psql'
 
 # add in the host/port, if they were specified (not null, -n)
@@ -58,11 +65,11 @@ fi
 if [ "$MIMIC_USER" != "postgres" ]; then
     # we need to create this user via postgres
     # use SUDO to login as postgres
-    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER; CREATE USER $MIMIC_USER WITH PASSWORD '$MIMIC_PASSWORD';"
+    $PSQL -U postgres -d postgres -c "DROP USER IF EXISTS $MIMIC_USER_SQL; CREATE USER $MIMIC_USER_SQL WITH PASSWORD '$MIMIC_PASSWORD_SQL';"
 fi
 
 if [ "$MIMIC_DB" != "postgres" ]; then
   # drop and recreate the database
-  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB;"
-  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB OWNER $MIMIC_USER;"
-fi
+  $PSQL -U postgres -d postgres -c "DROP DATABASE IF EXISTS $MIMIC_DB_SQL;"
+  $PSQL -U postgres -d postgres -c "CREATE DATABASE $MIMIC_DB_SQL OWNER $MIMIC_USER_SQL;"
+fi

mimic-iii/buildmimic/sqlite/import.py

@@ -10,6 +10,34 @@
 CHUNKSIZE = 10 ** 6
 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME)
 
+# Column dtypes for tables that trigger pandas mixed-type warnings when
+# loaded with the default low_memory chunked inference (see #1237).
+# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql.
+TABLE_DTYPES = {
+    "chartevents": {
+        "WARNING": "Int64",
+        "ERROR": "Int64",
+        "RESULTSTATUS": "string",
+        "STOPPED": "string",
+    },
+    "datetimeevents": {
+        "WARNING": "Int64",
+        "ERROR": "Int64",
+        "RESULTSTATUS": "string",
+        "STOPPED": "string",
+    },
+    "inputevents_cv": {
+        "ORIGINALRATE": "float64",
+        "ORIGINALRATEUOM": "string",
+        "ORIGINALSITE": "string",
+    },
+    "noteevents": {
+        "CHARTTIME": "string",
+        "STORETIME": "string",
+        "ISERROR": "string",
+    },
+}
+
 
 def _table_name_from_csv(filename: str) -> str:
     """Derive SQL table name from a CSV path (literal suffix, not str.strip)."""
@@ -21,20 +49,39 @@ def _table_name_from_csv(filename: str) -> str:
     return name.lower()
 
 
+def _read_csv(path, table, **kwargs):
+    # low_memory=False avoids dtype re-inference across chunks; table-specific
+    # dtypes pin the columns that otherwise flip between numeric/object.
+    return pd.read_csv(
+        path,
+        index_col="ROW_ID",
+        low_memory=False,
+        dtype=TABLE_DTYPES.get(table),
+        **kwargs,
+    )
+
+
 if os.path.exists(DATABASE_NAME):
     msg = "File {} already exists.".format(DATABASE_NAME)
     print(msg)
     sys.exit()
 
+# Prefer .csv.gz when both exist for the same table; also load plain .csv
+# (import.sh already accepts both).
+files_by_table = {}
+for f in glob("*.csv"):
+    files_by_table[_table_name_from_csv(f)] = f
 for f in glob("*.csv.gz"):
+    files_by_table[_table_name_from_csv(f)] = f
+
+for table, f in sorted(files_by_table.items()):
     print("Starting processing {}".format(f))
-    table = _table_name_from_csv(f)
     if os.path.getsize(f) < THRESHOLD_SIZE:
-        df = pd.read_csv(f, index_col="ROW_ID")
+        df = _read_csv(f, table)
         df.to_sql(table, CONNECTION_STRING)
     else:
         # If the file is too large, let's do the work in chunks
-        for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
+        for chunk in _read_csv(f, table, chunksize=CHUNKSIZE):
             chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
     print("Finished processing {}".format(f))
 

mimic-iii/buildmimic/sqlite/import.sh

@@ -28,19 +28,19 @@ for FILE in *; do
     TABLE_NAME=$(echo "${FILE%%.*}" | tr "[:upper:]" "[:lower:]")
     case "$FILE" in
         *csv)
-            IMPORT_CMD=".import $FILE $TABLE_NAME"
+            IMPORT_CMD=".import \"$FILE\" $TABLE_NAME"
         ;;
         # need to decompress csv before load
         *csv.gz)
-            IMPORT_CMD=".import \"|gzip -dc $FILE\" $TABLE_NAME"
+            IMPORT_CMD=".import \"|gzip -dc \\\"$FILE\\\"\" $TABLE_NAME"
         ;;
         # not a data file so skip
         *)
             continue
         ;;
     esac
     echo "Loading $FILE."
-    sqlite3 $OUTFILE <<EOF
+    sqlite3 "$OUTFILE" <<EOF
 .headers on
 .mode csv
 $IMPORT_CMD

mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37.sql

@@ -384,8 +384,10 @@ select hadm_id, seq_num, icd9_code
   when icd9_code = '3091' then 1
   when icd9_code = '311'         then 1
 		end as depress  /* Depression */
-from `physionet-data.mimiciii_clinical.diagnoses_icd` icd
-WHERE seq_num = 1
+from (
+  select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code
+  from `physionet-data.mimiciii_clinical.diagnoses_icd`
+) icd
 )
 -- collapse the icd9_code specific flags into hadm_id specific flags
 -- this groups comorbidities together for a single patient admission

mimic-iii/concepts/comorbidity/elixhauser_ahrq_v37_no_drg.sql

@@ -390,8 +390,11 @@ select hadm_id, seq_num, icd9_code
   when icd9_code = '3091' then 1
   when icd9_code = '311'         then 1
 		end as depress  /* Depression */
-from `physionet-data.mimiciii_clinical.diagnoses_icd` icd
-WHERE seq_num = 1
+from (
+  select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code
+  from `physionet-data.mimiciii_clinical.diagnoses_icd`
+  where seq_num != 1
+) icd
 )
 -- collapse the icd9_code specific flags into hadm_id specific flags
 -- this groups comorbidities together for a single patient admission

mimic-iii/concepts/comorbidity/elixhauser_quan.sql

@@ -174,7 +174,10 @@ select hadm_id, seq_num, icd9_code
   when SUBSTR(icd9_code, 1, 4) in ('2962','2963','2965','3004') then 1
   when SUBSTR(icd9_code, 1, 3) in ('309','311') then 1
   else 0 end as depress  /* Depression */
-from `physionet-data.mimiciii_clinical.diagnoses_icd` icd
+from (
+  select hadm_id, seq_num, RTRIM(icd9_code) as icd9_code
+  from `physionet-data.mimiciii_clinical.diagnoses_icd`
+) icd
 where seq_num != 1 -- we do not include the primary icd-9 code
 )
 -- collapse the icd9_code specific flags into hadm_id specific flags

mimic-iii/concepts/durations/adenosine_durations.sql

@@ -198,11 +198,39 @@ and
 (
   select
     icustay_id, linkorderid
-    , min(starttime) as starttime, max(endtime) as endtime
+    -- Keep each MV row: min/max by linkorderid includes pause gaps (#1808).
+    , starttime, endtime
   FROM `physionet-data.mimiciii_clinical.inputevents_mv`
   where itemid = 221282 -- adenosine
   and statusdescription != 'Rewritten' -- only valid orders
-  group by icustay_id, linkorderid
+)
+
+, vasomv_grp as
+(
+  -- Merge overlapping/abutting intervals; pause gaps stay separate (#1808).
+  SELECT
+    s1.icustay_id
+    , s1.starttime
+    , MIN(t1.endtime) AS endtime
+  FROM vasomv s1
+  INNER JOIN vasomv t1
+    ON s1.icustay_id = t1.icustay_id
+    AND s1.starttime <= t1.endtime
+    AND NOT EXISTS
+    (
+      SELECT 1 FROM vasomv t2
+      WHERE t1.icustay_id = t2.icustay_id
+        AND t1.endtime >= t2.starttime
+        AND t1.endtime < t2.endtime
+    )
+  WHERE NOT EXISTS
+  (
+    SELECT 1 FROM vasomv s2
+    WHERE s1.icustay_id = s2.icustay_id
+      AND s1.starttime > s2.starttime
+      AND s1.starttime <= s2.endtime
+  )
+  GROUP BY s1.icustay_id, s1.starttime
 )
 
 select
@@ -224,6 +252,6 @@ select
   , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours
   -- add durations
 from
-  vasomv
+  vasomv_grp
 
 order by icustay_id, vasonum;

mimic-iii/concepts/durations/arterial_line_durations.sql

@@ -163,16 +163,45 @@ with mv as
   group by icustay_id, arterial_line_rownum
   having min(charttime) != max(charttime)
 )
+-- collapse overlapping MetaVision arterial line intervals (#371)
+, mv_dur as
+(
+  SELECT
+    s1.icustay_id
+    , s1.starttime
+    , MIN(t1.endtime) AS endtime
+    , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours
+  FROM mv s1
+  INNER JOIN mv t1
+    ON s1.icustay_id = t1.icustay_id
+    AND s1.arterial_line = 1
+    AND t1.arterial_line = 1
+    AND s1.starttime <= t1.endtime
+    AND NOT EXISTS
+    (
+      SELECT 1 FROM mv t2
+      WHERE t1.icustay_id = t2.icustay_id
+        AND t2.arterial_line = 1
+        AND t1.endtime >= t2.starttime
+        AND t1.endtime < t2.endtime
+    )
+  WHERE s1.arterial_line = 1
+    AND NOT EXISTS
+    (
+      SELECT 1 FROM mv s2
+      WHERE s1.icustay_id = s2.icustay_id
+        AND s2.arterial_line = 1
+        AND s1.starttime > s2.starttime
+        AND s1.starttime <= s2.endtime
+    )
+  GROUP BY s1.icustay_id, s1.starttime
+)
 select icustay_id
   -- , arterial_line_rownum
   , starttime, endtime, duration_hours
 from cv_dur
 UNION ALL
---TODO: collapse metavision durations if they overlap
 select icustay_id
-  -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as arterial_line_rownum
-  , starttime, endtime
-  , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours
-from mv
-where arterial_line = 1
+  , starttime, endtime, duration_hours
+from mv_dur
 order by icustay_id, starttime;

mimic-iii/concepts/durations/central_line_durations.sql

@@ -158,16 +158,45 @@ with mv as
   group by icustay_id, central_line_rownum
   having min(charttime) != max(charttime)
 )
+-- collapse overlapping MetaVision central line intervals (#371)
+, mv_dur as
+(
+  SELECT
+    s1.icustay_id
+    , s1.starttime
+    , MIN(t1.endtime) AS endtime
+    , DATETIME_DIFF(MIN(t1.endtime), s1.starttime, HOUR) AS duration_hours
+  FROM mv s1
+  INNER JOIN mv t1
+    ON s1.icustay_id = t1.icustay_id
+    AND s1.central_line = 1
+    AND t1.central_line = 1
+    AND s1.starttime <= t1.endtime
+    AND NOT EXISTS
+    (
+      SELECT 1 FROM mv t2
+      WHERE t1.icustay_id = t2.icustay_id
+        AND t2.central_line = 1
+        AND t1.endtime >= t2.starttime
+        AND t1.endtime < t2.endtime
+    )
+  WHERE s1.central_line = 1
+    AND NOT EXISTS
+    (
+      SELECT 1 FROM mv s2
+      WHERE s1.icustay_id = s2.icustay_id
+        AND s2.central_line = 1
+        AND s1.starttime > s2.starttime
+        AND s1.starttime <= s2.endtime
+    )
+  GROUP BY s1.icustay_id, s1.starttime
+)
 select icustay_id
   -- , central_line_rownum
   , starttime, endtime, duration_hours
 from cv_dur
 UNION ALL
---TODO: collapse metavision durations if they overlap
 select icustay_id
-  -- , ROW_NUMBER() over (PARTITION BY icustay_id ORDER BY starttime) as central_line_rownum
-  , starttime, endtime
-  , DATETIME_DIFF(endtime, starttime, HOUR) AS duration_hours
-from mv
-where central_line = 1
+  , starttime, endtime, duration_hours
+from mv_dur
 order by icustay_id, starttime;