Run two-stages backups (#1)

1. Download to a local directory, archive/encrypt there
1. Push encrypted archive to remote folder

This helps when the backup destination is e.g. a NFS drive.

Also, introduce harder checks and fix some flaws.

Reviewed-on: #1
Co-authored-by: Max Mehl <mail@mehl.mx>
Co-committed-by: Max Mehl <mail@mehl.mx>

README.md

@@ -63,6 +63,21 @@ shortcuts:
 You can give multiple locations that shall be backed up. Just separate them by
 `|` characters. See the example file for more.
 
+## Process
+
+The script runs the following most important steps:
+1. For each host in `hosts.csv`
+    1. Check SSH connection
+    1. Compose SSH host settings
+    1. For each backup source
+        1. Resolve special backup sources
+        1. Create backup destination
+        1. rsync source to destination
+        1. tar the destination
+        1. gpg-encrypt the destination
+        1. Delete older backups
+1. Output completion info
+
 ## Manual run
 
 You can run `ssh-checker.sh` and `uberspace-backup.sh` manually. Without any arguments given, both will check/backup all hosts.

config.cfg.sample

@@ -4,10 +4,13 @@
 # File with hosts and their backup source paths
 HOSTS="$CURDIR"/hosts.csv
 
-# root dir where backups shall be saved to
-BACKUPDIR=/var/backups/uberspace
+# Temporary download destination for backups
+TEMPDIR=/tmp/uberspace-backup
 
-# GPG fingerprint of key used for encryption  
+# root dir where backups shall be saved to
+BACKUPDIR=/mnt/remotesrv/uberspace
+
+# GPG fingerprint of key used for encryption
 GPG=6775E8DDD8CEABCC83E38CEHE6334BCA29DF8192
 
 # Maximum number of backups that shall be retained (0 to disable automatic deletion)

ssh-checker.sh

@@ -68,6 +68,4 @@ while read -r line; do
     echo "[SUCCESS] SSH login possible for ${RHOST}."
   fi
 
-  echo
-
 done < "$HOSTS"

uberspace-backup.sh

@@ -9,6 +9,9 @@
 #
 ########################################################################
 
+# Fail fast on errors
+set -Eeuo pipefail
+
 # Set correct UTF-8 encoding (for FreeBSD jail)
 export LC_ALL=en_US.UTF-8
 
@@ -26,8 +29,6 @@ else
   SSH_KEY=~/.ssh/id_rsa
 fi
 
-ARG1="$1"
-
 # Get current date
 DATE=$(date +"%Y-%m-%d_%H-%M")
 LOG="$CURDIR"/backup.log
@@ -41,8 +42,8 @@ function pdate {
 }
 function logecho {
   # Echo string and copy it to log while attaching the current date
-  echo "$(pdate) $@"
-  echo "$(pdate) $@" >> "$LOG"
+  echo "$(pdate) $*"
+  echo "$(pdate) $*" >> "$LOG"
 }
 
 while read -r line; do
@@ -51,8 +52,8 @@ while read -r line; do
 
   RHOST=$(echo "$line" | cut -d";" -f1 | trim)
 
-  # Jump to next line if this line's host does not match host of ARG1 (if given)
-  if [[ "${ARG1}" != "" ]] && [[ "${ARG1}" != "${RHOST}" ]]; then
+  # Jump to next line if this line's host does not match host of first argument (if given)
+  if [[ "${1-}" != "" ]] && [[ "${1-}" != "${RHOST}" ]]; then
     continue
   fi
   # Task ssh-checker.sh to check this host
@@ -79,28 +80,37 @@ while read -r line; do
 
   logecho "${RHOST}: Starting backups"
 
-  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l)
+  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l || true)
   NORDIR=$(($NORDIR + 1))
 
+  # Loop through all backup sources
   for ((i = 1; i <= $NORDIR; i++)); do
     RDIR=$(echo "$ALLRDIR" | cut -d"|" -f${i} | trim)
 
+    # Set a relative destination directory
     if [ "${RDIR}" == "%virtual" ]; then
       RDIR=/var/www/virtual/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/virtual"
+      DEST_REL="$RHOST/$DATE/virtual"
     elif [ "${RDIR}" == "%mysql" ]; then
       RDIR=mysql
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     elif [ "${RDIR}" == "%mails" ]; then
       RDIR=/home/${RUSER}/users
-      DEST="$BACKUPDIR/$RHOST/$DATE/mails"
+      DEST_REL="$RHOST/$DATE/mails"
     elif [ "${RDIR}" == "%home" ]; then
       RDIR=/home/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/home"
+      DEST_REL="$RHOST/$DATE/home"
     else
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     fi
 
+    # Define absolute temporary and final backup destination paths
+    # Example:
+    #   DEST=/tmp/uberspace-backup/user@example.com/2019-01-01/virtual
+    #   DEST_FINAL=/media/Uberspace/user@example.com/2019-01-01/
+    DEST="${TEMPDIR}/${DEST_REL}"
+    DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
+
     # Set Source directory, and make exception for %mysql
     SOURCE="${RDIR}"
     if [ "${RDIR}" == "mysql" ]; then
@@ -111,8 +121,9 @@ while read -r line; do
       fi
     fi
 
-    # Create backup destination if necessary
+    # Create temporary and final backup destination if necessary
     if [ ! -e "${DEST}" ]; then mkdir -p "${DEST}"; fi
+    if [ ! -e "${DEST_FINAL}" ]; then mkdir -p "${DEST_FINAL}"; fi
 
     # RSYNC
     logecho "${RHOST}: Downloading ${SOURCE} to ${DEST}"
@@ -128,12 +139,19 @@ while read -r line; do
     gpg --output "${DEST}".tar.gpg --encrypt --recipient ${GPG} "${DEST}".tar
     rm "${DEST}".tar
 
-    # Delete all old directories except the $MAXBAK most recent
-    if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-      logecho "${RHOST}: Removing older backups of $(basename "${DEST}")"
-      ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
-    fi
-  done
+    # Push encrypted backup to final backup destination
+    logecho "${RHOST}: Moving $(basename "${DEST}") to ${DEST_FINAL}"
+    cp "${DEST}".tar.gpg "${DEST_FINAL}/"
+    rm "${DEST}".tar.gpg
+
+  done # End of loop through all backup sources
+
+  # Delete all old directories except the $MAXBAK most recent
+  if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
+    oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
+    logecho "${RHOST}: Removing older backup directories: ${oldbackups}"
+    ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
+  fi
 
 done < "$HOSTS"