diff --git a/Dockerfile b/Dockerfile index 716b77d..35a90e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,11 @@ RUN apk add --no-cache \ su-exec \ dcron \ gzip \ - && mkdir -p /scripts /backups + && mkdir -p /scripts COPY backup_full.sh /scripts/backup_full.sh COPY backup_incremental.sh /scripts/backup_incremental.sh +COPY backup_incremental_base.sh /scripts/backup_incremental_base.sh COPY entrypoint.sh /entrypoint.sh RUN chmod +x /scripts/*.sh /entrypoint.sh diff --git a/README.md b/README.md index 9e27bd7..1185f95 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ A simple, containerized solution for automated full and incremental backups of a ## Features -- **Full Backups:** Scheduled `pg_dump` backups of your PostgreSQL database, compressed and stored in `/backups/full`. -- **Incremental Backups:** Optionally archive PostgreSQL WAL files for point-in-time recovery, stored in `/backups/incremental`. +- **Full Backups:** Scheduled `pg_dump` backups of your PostgreSQL database, compressed and stored in `/backups/full` or `/backups/$BACKUP_SUBDIR/full`. +- **Incremental Backups:** Optionally perform a physical base backup (`pg_basebackup`) and archive PostgreSQL WAL files for point-in-time recovery. Base backups are stored in `/backups/base` or `/backups/$BACKUP_SUBDIR/base`, and WAL incrementals in `/backups/incremental` or `/backups/$BACKUP_SUBDIR/incremental`. Incrementals are retained only as long as their corresponding base backup exists. - **Retention Policies:** Automatically remove old backups based on configurable retention periods. - **Configurable Scheduling:** Use environment variables to control backup intervals via cron. - **Easy Integration:** Designed to run as a Docker container, with minimal configuration. @@ -15,19 +15,21 @@ A simple, containerized solution for automated full and incremental backups of a ### Environment Variables -| Variable | Description | Default | -|-------------------------------|----------------------------------------------------------|------------------------| -| `POSTGRES_HOST` | PostgreSQL host | (required) | -| `POSTGRES_PORT` | PostgreSQL port | (required) | -| `POSTGRES_USER` | PostgreSQL user | (required) | -| `POSTGRES_DB` | PostgreSQL database name | (required) | -| `PGPASSWORD_FILE` | Path to file containing the PostgreSQL password | (required) | -| `ENABLE_INCREMENTAL` | Enable incremental (WAL) backups (`true`/`false`) | `true` | -| `BACKUP_NAME` | Name for the backup file | `backup` | -| `RETENTION_FULL_DAYS` | Days to keep full backups | `7` | -| `RETENTION_INC_DAYS` | Days to keep incremental backups | `3` | -| `BACKUP_FULL_INTERVAL` | Cron schedule for full backups | `0 2 * * 0` | -| `BACKUP_INCREMENTAL_INTERVAL` | Cron schedule for incremental backups | `0 */6 * * *` | +| Variable | Description | Default | +|-----------------------------------|----------------------------------------------------------|------------------------| +| `POSTGRES_HOST` | PostgreSQL host | (required) | +| `POSTGRES_PORT` | PostgreSQL port | (required) | +| `POSTGRES_USER` | PostgreSQL user | (required) | +| `POSTGRES_DB` | PostgreSQL database name | (required) | +| `PGPASSWORD_FILE` | Path to file containing the PostgreSQL password | (required) | +| `ENABLE_INCREMENTAL` | Enable incremental (WAL) backups (`true`/`false`) | `true` | +| `BACKUP_NAME` | Name for the backup file | `backup` | +| `RETENTION_FULL_DAYS` | Days to keep full backups | `7` | +| `RETENTION_INC_DAYS` | Days to keep incremental backups | `3` | +| `BACKUP_FULL_INTERVAL` | Cron schedule for full backups | `0 2 1 * *` | +| `BACKUP_INCREMENTAL_BASE_INTERVAL`| Cron schedule for incremental base backups | `0 3 * * 0` | +| `BACKUP_INCREMENTAL_INTERVAL` | Cron schedule for incremental backups | `0 */6 * * *` | +| `BACKUP_SUBDIR` | Subdirectory for backups to be stored | (undefined) | ### Volumes @@ -43,8 +45,8 @@ docker run -d \ -e POSTGRES_USER=postgres \ -e POSTGRES_DB=mydb \ -e PGPASSWORD_FILE=/run/secrets/pgpassword \ - -e RETENTION_FULL_DAYS=7 \ - -e RETENTION_INC_DAYS=3 \ + -e RETENTION_FULL_DAYS=30 \ + -e RETENTION_INC_DAYS=10 \ -e ENABLE_INCREMENTAL=true \ -v /host/backups:/backups \ -v /host/wal_archive:/wal_archive \ @@ -67,8 +69,8 @@ services: - POSTGRES_USER=postgres - POSTGRES_DB=mydb - PGPASSWORD_FILE=/run/secrets/pgpassword - - RETENTION_FULL_DAYS=7 - - RETENTION_INC_DAYS=3 + - RETENTION_FULL_DAYS=30 + - RETENTION_INC_DAYS=10 - ENABLE_INCREMENTAL=true volumes: - /host/backups:/backups diff --git a/backup_full.sh b/backup_full.sh index 910ccc3..f62972e 100755 --- a/backup_full.sh +++ b/backup_full.sh @@ -1,19 +1,34 @@ #!/bin/bash set -euo pipefail +BACKUP_SUBDIR="${BACKUP_SUBDIR:-}" +if [ -n "$BACKUP_SUBDIR" ]; then + BASE_BACKUP_DIR="/backups/$BACKUP_SUBDIR" +else + BASE_BACKUP_DIR="/backups" +fi + DATE=$(date +%F_%H-%M-%S) -DEST_DIR="/backups/full/$DATE" -mkdir -p "$DEST_DIR" +DEST_DIR="$BASE_BACKUP_DIR/full/$DATE" +if ! mkdir -p "$DEST_DIR"; then + echo "[ERROR] Failed to create directory $DEST_DIR" + exit 1 +fi export PGPASSWORD=$(cat $PGPASSWORD_FILE) -echo "[$(date)] Performing full backup..." -pg_dump -h "${POSTGRES_HOST}" \ +echo "[INFO] Performing full backup..." +if ! pg_dump -h "${POSTGRES_HOST}" \ -p "${POSTGRES_PORT}" \ -U "${POSTGRES_USER}" \ - "${POSTGRES_DB}" | gzip > "$DEST_DIR/$BACKUP_NAME.gz" + "${POSTGRES_DB}" | gzip > "$DEST_DIR/$BACKUP_NAME.gz"; then + echo "[ERROR] $1" + exit 1 +fi # Apply retention -find /backups/full -type d -mtime +${RETENTION_FULL_DAYS} -exec rm -rf {} + +if ! find "$BASE_BACKUP_DIR/full" -type d -mtime +"${RETENTION_FULL_DAYS}" -exec rm -rf {} +; then + echo "[WARNING] Retention cleanup failed" +fi -echo "[$(date)] Full backup completed." \ No newline at end of file +echo "[INFO] Full backup completed." \ No newline at end of file diff --git a/backup_incremental.sh b/backup_incremental.sh index 7732453..3d229de 100755 --- a/backup_incremental.sh +++ b/backup_incremental.sh @@ -1,24 +1,36 @@ #!/bin/bash set -euo pipefail +BACKUP_SUBDIR="${BACKUP_SUBDIR:-}" +if [ -n "$BACKUP_SUBDIR" ]; then + BASE_BACKUP_DIR="/backups/$BACKUP_SUBDIR" +else + BASE_BACKUP_DIR="/backups" +fi + DATE=$(date +%F_%H-%M-%S) -DEST_DIR="/backups/incremental/$DATE" -mkdir -p "$DEST_DIR" +DEST_DIR="$BASE_BACKUP_DIR/incremental/$DATE" +if ! mkdir -p "$DEST_DIR"; then + echo "[ERROR] Failed to create directory $DEST_DIR" + exit 1 +fi -echo "[$(date)] Performing incremental backup..." +echo "[INFO] Performing incremental backup..." # Backup all WALs since last backup -cp /wal_archive/* "$DEST_DIR/" || true - -# Apply retention on the incremental backups themselves -find /backups/incremental -type d -mtime +${RETENTION_INC_DAYS} -exec rm -rf {} + +if ! cp /wal_archive/* "$DEST_DIR/" 2>/dev/null; then + echo "[WARNING] No WAL files found to copy from /wal_archive" +fi # Clean up WAL files using pg_archivecleanup -# Determine the last WAL file to keep -LAST_WAL=$(ls -1 /wal_archive/* | sort | tail -n 1 || true) +LAST_WAL=$(ls -1 /wal_archive/* 2>/dev/null | sort | tail -n 1 || true) if [ -n "$LAST_WAL" ]; then - echo "[$(date)] Cleaning up WAL archive up to $LAST_WAL" - pg_archivecleanup /wal_archive "$LAST_WAL" + echo "[INFO] Cleaning up WAL archive up to $LAST_WAL" + if ! pg_archivecleanup /wal_archive "$LAST_WAL"; then + echo "[WARNING] pg_archivecleanup failed" + fi +else + echo "[INFO] No WAL files found for cleanup." fi -echo "[$(date)] Incremental backup completed." \ No newline at end of file +echo "[INFO] Incremental backup completed." \ No newline at end of file diff --git a/backup_incremental_base.sh b/backup_incremental_base.sh new file mode 100755 index 0000000..cade788 --- /dev/null +++ b/backup_incremental_base.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -euo pipefail + +BACKUP_SUBDIR="${BACKUP_SUBDIR:-}" +if [ -n "$BACKUP_SUBDIR" ]; then + BASE_BACKUP_DIR="/backups/$BACKUP_SUBDIR" +else + BASE_BACKUP_DIR="/backups" +fi + +DATE=$(date +%F_%H-%M-%S) +DEST_DIR="$BASE_BACKUP_DIR/incremental_base/$DATE" + +echo "[INFO] Performing incremental base backup into $DEST_DIR" +mkdir -p "$DEST_DIR" + +export PGPASSWORD=$(cat "$PGPASSWORD_FILE") + +if ! pg_basebackup \ + -h "${POSTGRES_HOST}" \ + -p "${POSTGRES_PORT}" \ + -U "${POSTGRES_USER}" \ + -D "$DEST_DIR" \ + -F tar \ + -z \ + -X none; then + echo "[ERROR] Base backup failed" + exit 1 +fi + +echo "[INFO] Base backup completed." + +# Cleanup old incremental base backups +echo "[INFO] Applying incremental base backup retention policy..." +if ! find "$BASE_BACKUP_DIR/incremental_base" -mindepth 1 -maxdepth 1 -type d -mtime +"${RETENTION_INC_DAYS}" -print -exec rm -rf {} + 2>&1; then + echo "[WARNING] Retention cleanup for incremental base backups may have failed" >&2 +fi + +# Cleanup old incrementals relative to the oldest base +OLDEST_BASE=$(ls -1 "$BASE_BACKUP_DIR/incremental_base" | sort | head -n 1 || true) + +if [ -n "$OLDEST_BASE" ]; then + echo "[INFO] Retaining incrementals since base $OLDEST_BASE" + if ! find "$BASE_BACKUP_DIR/incremental" -mindepth 1 -maxdepth 1 -type d \ + ! -newer "$BASE_BACKUP_DIR/incremental_base/$OLDEST_BASE" \ + -print -exec rm -rf {} + 2>&1; then + echo "[WARNING] Retention cleanup for incrementals may have failed" >&2 + fi +else + echo "[INFO] No incremental base backups found, skipping incremental cleanup" +fi diff --git a/entrypoint.sh b/entrypoint.sh index 735518e..dfdd346 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -5,14 +5,16 @@ CRON_DIR=/tmp/cron mkdir -p "$CRON_DIR" ENABLE_INCREMENTAL="${ENABLE_INCREMENTAL:-true}" -FULL_INTERVAL="${BACKUP_FULL_INTERVAL:-0 2 * * 0}" +FULL_INTERVAL="${BACKUP_FULL_INTERVAL:-0 2 1 * *}" # Default to 2:00 AM on the first day of each month cat > "$CRON_DIR/backup" <> "$CRON_DIR/backup" + INC_INTERVAL="${BACKUP_INCREMENTAL_INTERVAL:-0 */6 * * *}" # Default to every 6 hours echo "$INC_INTERVAL /scripts/backup_incremental.sh" >> "$CRON_DIR/backup" fi