Enhance dtn_download.sh for improved usability and logging

- Updated usage instructions to include a dry run option and clarified parallel job settings.
- Changed default parallel jobs from 8 to 4 for better performance.
- Added logging capabilities to track transfer jobs and their outputs.
- Improved user feedback during directory discovery and transfer completion.

Author: FridrichMethod

scripts/gromacs/data_transfer/sherlock/dtn_download.sh

@@ -3,17 +3,30 @@
 set -euo pipefail
 
 usage() {
-    echo "Usage: ${0##*/} [-j <n>] REMOTE_DIR LOCAL_DIR" >&2
+    cat >&2 <<EOF
+Usage: ${0##*/} [-j <n>] [-n] REMOTE_DIR LOCAL_DIR
+
+Options:
+    -j <n>   Number of parallel rsync jobs (default: 4)
+    -n       Dry run (show what would be transferred)
+
+Example:
+    ${0##*/} -j 4 /scratch/users/\$USER/md_runs /data/local/md_runs
+EOF
     exit 1
 }
 
 LOGIN_HOST="sherlock-plain"
 DTN_HOST="sherlock-dtn"
-JOBS=8
+JOBS=4
+DRY_RUN=""
+
+SSH_OPTS="ssh -T -c aes128-gcm@openssh.com -o Compression=no -o ServerAliveInterval=60"
 
-while getopts "j:" opt; do
+while getopts "j:n" opt; do
     case $opt in
         j) JOBS="$OPTARG" ;;
+        n) DRY_RUN="--dry-run" ;;
         *) usage ;;
     esac
 done
@@ -25,6 +38,7 @@ LOCAL_DIR="$2"
 
 mkdir -p "$LOCAL_DIR"
 
+echo "=== Discovering subdirectories in $REMOTE_DIR ==="
 mapfile -t SUBDIRS < <(
     ssh -n "$LOGIN_HOST" "find \"$REMOTE_DIR\" -mindepth 1 -maxdepth 1 -type d -printf '%f\n'" | sort
 )
@@ -33,8 +47,17 @@ mapfile -t SUBDIRS < <(
     exit 1
 }
 
-parallel -j "$JOBS" \
-    rsync -ahP --append-verify \
+echo "=== Found ${#SUBDIRS[@]} subdirectories, transferring with $JOBS parallel jobs ==="
+
+LOGDIR="$LOCAL_DIR/.transfer_logs"
+mkdir -p "$LOGDIR"
+
+parallel -j "$JOBS" --bar --joblog "$LOGDIR/joblog.tsv" \
+    rsync -ahP --append-verify $DRY_RUN \
+    -e "'$SSH_OPTS'" \
+    --log-file="'$LOGDIR/{}.log'" \
     "$DTN_HOST:$REMOTE_DIR/{}/" \
     "$LOCAL_DIR/{}/" \
     ::: "${SUBDIRS[@]}"
+
+echo "=== Transfer complete. Logs in $LOGDIR ==="