chore: enhance update.sh with configurable directories, memory, and threads; add new test cases and improved cleanup logic

Author: dgraf-gh

scripts/update.sh

@@ -15,6 +15,11 @@ PBF_FILTERED_FILE="planet-filtered.pbf"
 IMPORT_DIR="import"
 DOCKER_IMAGE="dedicatedcode/paikka:develop"
 
+DOWNLOAD_DIR="${DOWNLOAD_DIR:-$LOCAL_WORK_DIR}"
+IMPORT_DATA_DIR="${IMPORT_DATA_DIR:-$LOCAL_WORK_DIR/$IMPORT_DIR}"
+IMPORT_MEMORY="${IMPORT_MEMORY:-16G}"
+IMPORT_THREADS="${IMPORT_THREADS:-10}"
+
 # --- Remote Machine Settings ---
 REMOTE_BASE_DIR="/opt/paikka/data"
 
@@ -24,7 +29,9 @@ GEOCODER_TEST_URL_BASE="http://localhost:8080/v1/reverse"
 
 # --- Verification Test Cases ---
 declare -A TEST_CASES=(
-  ["lat=52.516280&lon=13.377635"]="518071791"     # Brandenburger Tor
+  ["lat=52.516280&lon=13.377635"]="518071791" # Brandenburger Tor
+  ["lat=48.85826&lon=2.2945008"]="5013364"    # Eiffel Tower
+  ["lat=40.68924&lon=-74.044502"]="32965412"  # Statue of Liberty
 )
 
 # Global variables that will be set by parse_args_and_configure or environment
@@ -56,31 +63,42 @@ parse_args_and_configure() {
     REMOTE_HOST="${2:-$REMOTE_HOST}"
     GEOCODER_API_TOKEN="${3:-$GEOCODER_API_TOKEN}"
 
+    DOWNLOAD_DIR="${4:-$DOWNLOAD_DIR}"
+    IMPORT_DATA_DIR="${5:-$IMPORT_DATA_DIR}"
+    IMPORT_MEMORY="${6:-$IMPORT_MEMORY}"
+    IMPORT_THREADS="${7:-$IMPORT_THREADS}"
+
     if [ -z "$REMOTE_USER" ] || [ -z "$REMOTE_HOST" ] || [ -z "$GEOCODER_API_TOKEN" ]; then
-        echo "Usage: $0 <REMOTE_USER> <REMOTE_HOST> <API_TOKEN>"
+        echo "Usage: $0 <REMOTE_USER> <REMOTE_HOST> <API_TOKEN> [DOWNLOAD_DIR] [IMPORT_DATA_DIR] [MEMORY] [THREADS]"
+        echo "  DOWNLOAD_DIR: Where to download PBF files (default: current directory)"
+        echo "  IMPORT_DATA_DIR: Where to store import data (default: ./import)"
+        echo "  MEMORY: Memory for import (default: 16G)"
+        echo "  THREADS: Threads for import (default: 10)"
         echo "Error: Missing required configuration."
-        echo "Provide arguments or set them as environment variables (REMOTE_USER, REMOTE_HOST, GEOCODER_API_TOKEN)."
         exit 1
     fi
-    # Set a dynamic variable that depends on other config being set
-    ZIP_FILENAME="paikka_import_$(date +%Y%m%d_%H%M%S).zip"
     echo "Configuration loaded for ${REMOTE_USER}@${REMOTE_HOST}"
-}
+    echo "  Download directory: $DOWNLOAD_DIR"
+    echo "  Import data directory: $IMPORT_DATA_DIR"
+    echo "  Import memory: $IMPORT_MEMORY"
+    echo "  Import threads: $IMPORT_THREADS"}
 
 ###
 # LOCAL: Creates the necessary working directories.
 ###
 local_prepare_directories() {
     log "LOCAL: Ensuring import directory exists"
-    mkdir -p "$LOCAL_WORK_DIR/$IMPORT_DIR"
-    cd "$LOCAL_WORK_DIR"
+    mkdir -p "$DOWNLOAD_DIR"
+    mkdir -p "$IMPORT_DATA_DIR"
+    cd "$DOWNLOAD_DIR"
 }
 
 ###
 # LOCAL: Downloads the latest OSM planet file.
 ###
 local_download_planet_file() {
-    log "LOCAL: Downloading latest OSM planet file (if newer)"
+    log "LOCAL: Downloading latest OSM planet file to $DOWNLOAD_DIR"
+    cd "$DOWNLOAD_DIR"
     wget -N "$PLANET_URL"
 }
 
@@ -97,53 +115,50 @@ local_pull_docker_image() {
 ###
 local_filter_pbf() {
     log "LOCAL: Filtering PBF file (approx. 50 mins)"
-    sudo docker run --rm -v "$(pwd)":/data "$DOCKER_IMAGE" prepare "$PBF_INPUT_FILE" "$PBF_FILTERED_FILE"
+    cd "$DOWNLOAD_DIR"
+    sudo docker run --rm -ti -v "$DOWNLOAD_DIR":/data "$DOCKER_IMAGE" prepare "$PBF_INPUT_FILE" "$PBF_FILTERED_FILE"
 }
 
 ###
 # LOCAL: Creates the geocoder import bundle from the filtered PBF.
 ###
 local_create_import_bundle() {
-    log "LOCAL: Creating import bundle (approx. 15 hours)"
-    sudo docker run --rm -ti -v "$(pwd)":/data "$DOCKER_IMAGE" import \
-      --memory 16G \
-      --threads 10 \
-      --data-dir "/data/$IMPORT_DIR/" \
-      "$PBF_FILTERED_FILE"
+    log "LOCAL: Creating import bundle (approx. 15 hours) with $IMPORT_MEMORY memory and $IMPORT_THREADS threads"
+        sudo docker run --rm -ti -v "$DOWNLOAD_DIR":/download -v "$IMPORT_DATA_DIR":/import "$DOCKER_IMAGE" import \
+          --memory "$IMPORT_MEMORY" \
+          --threads "$IMPORT_THREADS" \
+          --data-dir "/import/" \
+          "/download/$PBF_FILTERED_FILE"
 }
 
 ###
 # LOCAL: Removes the large, intermediate PBF files.
 ###
 local_cleanup_pbf() {
-    log "LOCAL: Cleaning up intermediate PBF files"
+    log "LOCAL: Cleaning up intermediate PBF files from $DOWNLOAD_DIR"
+    cd "$DOWNLOAD_DIR"
     rm -f "$PBF_FILTERED_FILE" "$PBF_INPUT_FILE"
-    echo "Deleted '$PBF_FILTERED_FILE' and '$PBF_INPUT_FILE'"
+    echo "Deleted '$DOWNLOAD_DIR/$PBF_FILTERED_FILE' and '$DOWNLOAD_DIR/$PBF_INPUT_FILE'"
 }
 ###
 # REMOTE: Syncs the import directory to the remote server using rsync.
 # Uses --link-dest to minimize bandwidth and remote disk usage.
 ###
 remote_sync_bundle() {
-    log "REMOTE: Syncing bundle via rsync (Delta transfer)"
+      log "REMOTE: Syncing bundle via rsync (Delta transfer)"
 
-    # We need to know the current live directory to use it as a link-dest
-    CURRENT_LIVE=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "readlink -f ${REMOTE_BASE_DIR}/live_data || true")
-    NEW_RELEASE_TIMESTAMP=$(date +%Y%m%d%H%M%S)
-    NEW_RELEASE_DIR="${REMOTE_BASE_DIR}/releases/${NEW_RELEASE_TIMESTAMP}"
+      CURRENT_LIVE=$(ssh "${REMOTE_USER}@${REMOTE_HOST}" "readlink -f ${REMOTE_BASE_DIR}/live_data || true")
+      NEW_RELEASE_TIMESTAMP=$(date +%Y%m%d%H%M%S)
+      NEW_RELEASE_DIR="${REMOTE_BASE_DIR}/releases/${NEW_RELEASE_TIMESTAMP}"
 
-    # Ensure the remote releases directory exists
-    ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_BASE_DIR}/releases"
+      ssh "${REMOTE_USER}@${REMOTE_HOST}" "mkdir -p ${REMOTE_BASE_DIR}/releases"
 
-    # --link-dest makes rsync hard-link unchanged files from the current release
-    # into the new release directory, saving bandwidth and space.
-    rsync -avz --progress \
-        ${CURRENT_LIVE:+--link-dest="$CURRENT_LIVE"} \
-        "$LOCAL_WORK_DIR/$IMPORT_DIR/" \
-        "${REMOTE_USER}@${REMOTE_HOST}:$NEW_RELEASE_DIR/"
+      rsync -avz --progress \
+          ${CURRENT_LIVE:+--link-dest="$CURRENT_LIVE"} \
+          "$IMPORT_DATA_DIR/" \
+          "${REMOTE_USER}@${REMOTE_HOST}:$NEW_RELEASE_DIR/"
 
-    # Export this for the next step
-    export LATEST_RELEASE_DIR_NAME="$NEW_RELEASE_TIMESTAMP"
+      export LATEST_RELEASE_DIR_NAME="$NEW_RELEASE_TIMESTAMP"
 }
 
 ###
@@ -152,6 +167,12 @@ remote_sync_bundle() {
 remote_deploy_and_verify() {
     log "REMOTE: Executing remote deployment (Atomic Swap)"
 
+    # Convert TEST_CASES to a format that can be passed to remote shell
+    local test_cases_str=""
+    for key in "${!TEST_CASES}"; do
+        test_cases_str+="[\"$key\"]=\"${TEST_CASES[$key]}\" "
+    done
+
     ssh "${REMOTE_USER}@${REMOTE_HOST}" /bin/bash << EOF
   set -e
   BASE_DIR="${REMOTE_BASE_DIR}"
@@ -161,26 +182,34 @@ remote_deploy_and_verify() {
   NEW_RELEASE_DIR="releases/${LATEST_RELEASE_DIR_NAME}"
   LIVE_DATA_SYMLINK="live_data"
 
+  # Define TESTS array on remote side
+  declare -A TESTS=($test_cases_str)
+
+  echo_remote() {
+    echo "[REMOTE] \$1"
+  }
+
   cd "\$BASE_DIR"
 
   OLD_RELEASE_DIR=""
   [ -L "\$LIVE_DATA_SYMLINK" ] && OLD_RELEASE_DIR=\$(readlink \$LIVE_DATA_SYMLINK)
 
-  echo "Switching symlink: \$LIVE_DATA_SYMLINK -> \$NEW_RELEASE_DIR"
+  echo_remote "Switching symlink: \$LIVE_DATA_SYMLINK -> \$NEW_RELEASE_DIR"
   ln -sfn "\$NEW_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
 
-  echo "Refreshing Geocoder DB..."
+  echo_remote "Refreshing Geocoder DB..."
   HTTP_STATUS=\$(curl -s -o /dev/null -w "%{http_code}" -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL")
 
   if [ "\$HTTP_STATUS" -ne 200 ]; then
-      echo "ERROR: Refresh failed (\$HTTP_STATUS). Rolling back."
+      echo_remote "ERROR: Refresh failed (\$HTTP_STATUS). Rolling back."
       [ -n "\$OLD_RELEASE_DIR" ] && ln -sfn "\$OLD_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
       exit 1
   fi
- # --- 2. Verify ---
+
+  # --- 2. Verify ---
   echo_remote "Verifying new data..."
   VERIFICATION_FAILED=0
-  for query in "\${!TESTS[@]}"; do
+  for query in "\${!TESTS}"; do
     ACTUAL_ID=\$(curl -s "\$TEST_URL_BASE?\$query" | jq -r '.[0].id // "not_found"')
     if [ "\$ACTUAL_ID" != "\${TESTS[\$query]}" ]; then
       echo_remote "  --> FAILED: For \$query, expected '\${TESTS[\$query]}', got '\$ACTUAL_ID'"
@@ -195,7 +224,7 @@ remote_deploy_and_verify() {
     echo_remote "VERIFICATION FAILED. Rolling back and re-refreshing."
     if [ -n "\$OLD_RELEASE_DIR" ] && [ -d "\$OLD_RELEASE_DIR" ]; then
       ln -sfn "\$OLD_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
-      curl -s -o /dev/null -X POST -H "Authorization: Bearer \$API_TOKEN" "\$ADMIN_URL"
+      curl -s -o /dev/null -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL"
       echo_remote "Rollback to \$OLD_RELEASE_DIR complete. Faulty data in \$NEW_RELEASE_DIR is kept for inspection."
       exit 1
     else
@@ -205,17 +234,54 @@ remote_deploy_and_verify() {
   else
     echo_remote "VERIFICATION SUCCEEDED. Cleaning up old release and archive."
     [ -n "\$OLD_RELEASE_DIR" ] && [ -d "\$OLD_RELEASE_DIR" ] && rm -rf "\$OLD_RELEASE_DIR"
-    rm "\$ZIP_FILENAME"
     echo_remote "Deployment successful."
   fi
-
-  if [ \$VERIFICATION_FAILED -eq 0 ]; then
-    echo "Success. Cleaning up old release..."
-    [ -n "\$OLD_RELEASE_DIR" ] && [ "\$OLD_RELEASE_DIR" != "\$NEW_RELEASE_DIR" ] && rm -rf "\$OLD_RELEASE_DIR"
-  fi
 EOF
 }
 
+remote_cleanup_old_releases() {
+    log "REMOTE: Cleaning up old releases (keeping last 3 successful ones)"
+
+    ssh "${REMOTE_USER}@${REMOTE_HOST}" /bin/bash << EOF
+  set -e
+  BASE_DIR="${REMOTE_BASE_DIR}"
+
+  echo_remote() {
+    echo "[REMOTE CLEANUP] \$1"
+  }
+
+  cd "\$BASE_DIR"
+
+  # Keep last 3 successful releases (excluding current live)
+  echo_remote "Finding old releases to clean up..."
+  CURRENT_LIVE=\$(readlink -f live_data 2>/dev/null || echo "")
+
+  # List all releases, sort by timestamp, exclude current live
+  RELEASES=\$(find releases -maxdepth 1 -type d -name "[0-9]*" | sort -r)
+
+  KEEP_COUNT=3
+  COUNT=0
+  for release in \$RELEASES; do
+    if [ "\$release" = "\$CURRENT_LIVE" ] || [ "\$release" = "\$(basename "\$CURRENT_LIVE")" ]; then
+      echo_remote "Skipping current live release: \$release"
+      continue
+    fi
+
+    COUNT=\$((COUNT + 1))
+    if [ \$COUNT -gt \$KEEP_COUNT ]; then
+      echo_remote "Removing old release: \$release"
+      rm -rf "\$release"
+    else
+      echo_remote "Keeping release: \$release"
+    fi
+  done
+
+  # Also clean up any empty directories
+  find releases -type d -empty -delete 2>/dev/null || true
+
+  echo_remote "Cleanup complete"
+EOF
+}
 # ==============================================================================
 # MAIN ORCHESTRATION FUNCTION
 # ==============================================================================
@@ -231,7 +297,7 @@ main() {
     local_cleanup_pbf
     remote_sync_bundle
     remote_deploy_and_verify
-
+    remote_cleanup_old_releases
 
     log "Update process finished."
 }