make log file destination configurable

1. Download to a local directory, archive/encrypt there
1. Push encrypted archive to remote folder

This helps when the backup destination is e.g. a NFS drive.

Also, introduce harder checks and fix some flaws.

Reviewed-on: #1
Co-authored-by: Max Mehl <mail@mehl.mx>
Co-committed-by: Max Mehl <mail@mehl.mx>

README.md

@@ -63,6 +63,21 @@ shortcuts:
 You can give multiple locations that shall be backed up. Just separate them by
 `|` characters. See the example file for more.
 
+## Process
+
+The script runs the following most important steps:
+1. For each host in `hosts.csv`
+    1. Check SSH connection
+    1. Compose SSH host settings
+    1. For each backup source
+        1. Resolve special backup sources
+        1. Create backup destination
+        1. rsync source to destination
+        1. tar the destination
+        1. gpg-encrypt the destination
+        1. Delete older backups
+1. Output completion info
+
 ## Manual run
 
 You can run `ssh-checker.sh` and `uberspace-backup.sh` manually. Without any arguments given, both will check/backup all hosts.

config.cfg.sample

@@ -4,8 +4,11 @@
 # File with hosts and their backup source paths
 HOSTS="$CURDIR"/hosts.csv
 
+# Temporary download destination for backups
+TEMPDIR=/tmp/uberspace-backup
+
 # root dir where backups shall be saved to
-BACKUPDIR=/var/backups/uberspace
+BACKUPDIR=/mnt/remotesrv/uberspace
 
 # GPG fingerprint of key used for encryption
 GPG=6775E8DDD8CEABCC83E38CEHE6334BCA29DF8192
@@ -15,3 +18,6 @@ MAXBAK=3
 
 # SSH key
 #SSH_KEY="~/.ssh/mykey_rsa"
+
+# Logfile. Default: $CURDIR/backup.log
+# LOG_FILE=/var/log/uberspace-backup.log

ssh-checker.sh

@@ -68,6 +68,4 @@ while read -r line; do
     echo "[SUCCESS] SSH login possible for ${RHOST}."
   fi
 
-  echo
-
 done < "$HOSTS"

uberspace-backup.sh

@@ -9,9 +9,15 @@
 #
 ########################################################################
 
+# Fail fast on errors
+set -Eeuo pipefail
+
 # Set correct UTF-8 encoding (for FreeBSD jail)
 export LC_ALL=en_US.UTF-8
 
+# Initialise variables
+LOG_FILE=
+
 CURDIR=$(dirname "$(readlink -f "$0")")
 if [ ! -e "$CURDIR"/config.cfg ]; then echo "Missing config.cfg file. Edit and rename config.cfg.sample"; exit 1; fi
 source "$CURDIR"/config.cfg
@@ -26,11 +32,13 @@ else
   SSH_KEY=~/.ssh/id_rsa
 fi
 
-ARG1="$1"
+if [ -z "${LOG_FILE}" ]; then
+  # defaults
+  LOG_FILE="$CURDIR"/backup.log
+fi
 
 # Get current date
 DATE=$(date +"%Y-%m-%d_%H-%M")
-LOG="$CURDIR"/backup.log
 
 function trim {
   sed -r -e 's/^[[:space:]]*//g' -e 's/[[:space:]]*$//g'
@@ -41,18 +49,19 @@ function pdate {
 }
 function logecho {
   # Echo string and copy it to log while attaching the current date
-  echo "$(pdate) $@"
-  echo "$(pdate) $@" >> "$LOG"
+  echo "$(pdate) $*"
+  echo "$(pdate) $*" >> "$LOG_FILE"
 }
 
+# Loop over all hosts
 while read -r line; do
   # if line is a comment or blank, go to next line
   if echo "$line" | grep -qE "^\s*(#|$)"; then continue; fi
 
   RHOST=$(echo "$line" | cut -d";" -f1 | trim)
 
-  # Jump to next line if this line's host does not match host of ARG1 (if given)
-  if [[ "${ARG1}" != "" ]] && [[ "${ARG1}" != "${RHOST}" ]]; then
+  # Jump to next line if this line's host does not match host of first argument (if given)
+  if [[ "${1-}" != "" ]] && [[ "${1-}" != "${RHOST}" ]]; then
     continue
   fi
   # Task ssh-checker.sh to check this host
@@ -79,28 +88,40 @@ while read -r line; do
 
   logecho "${RHOST}: Starting backups"
 
-  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l)
+  logecho "${RHOST}: Deleting host's temporary directories in ${TEMPDIR}"
+  rm -rf "${TEMPDIR:?}/${RHOST:?}/"*
+
+  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l || true)
   NORDIR=$(($NORDIR + 1))
 
+  # Loop through all backup sources
   for ((i = 1; i <= $NORDIR; i++)); do
     RDIR=$(echo "$ALLRDIR" | cut -d"|" -f${i} | trim)
 
+    # Set a relative destination directory
     if [ "${RDIR}" == "%virtual" ]; then
       RDIR=/var/www/virtual/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/virtual"
+      DEST_REL="$RHOST/$DATE/virtual"
     elif [ "${RDIR}" == "%mysql" ]; then
       RDIR=mysql
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     elif [ "${RDIR}" == "%mails" ]; then
       RDIR=/home/${RUSER}/users
-      DEST="$BACKUPDIR/$RHOST/$DATE/mails"
+      DEST_REL="$RHOST/$DATE/mails"
     elif [ "${RDIR}" == "%home" ]; then
       RDIR=/home/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/home"
+      DEST_REL="$RHOST/$DATE/home"
     else
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     fi
 
+    # Define absolute temporary and final backup destination paths
+    # Example:
+    #   DEST=/tmp/uberspace-backup/user@example.com/2019-01-01/virtual
+    #   DEST_FINAL=/media/Uberspace/user@example.com/2019-01-01/
+    DEST="${TEMPDIR}/${DEST_REL}"
+    DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
+
     # Set Source directory, and make exception for %mysql
     SOURCE="${RDIR}"
     if [ "${RDIR}" == "mysql" ]; then
@@ -111,8 +132,9 @@ while read -r line; do
       fi
     fi
 
-    # Create backup destination if necessary
+    # Create temporary and final backup destination if necessary
     if [ ! -e "${DEST}" ]; then mkdir -p "${DEST}"; fi
+    if [ ! -e "${DEST_FINAL}" ]; then mkdir -p "${DEST_FINAL}"; fi
 
     # RSYNC
     logecho "${RHOST}: Downloading ${SOURCE} to ${DEST}"
@@ -128,12 +150,19 @@ while read -r line; do
     gpg --output "${DEST}".tar.gpg --encrypt --recipient ${GPG} "${DEST}".tar
     rm "${DEST}".tar
 
-    # Delete all old directories except the $MAXBAK most recent
-    if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-      logecho "${RHOST}: Removing older backups of $(basename "${DEST}")"
-      ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
-    fi
-  done
+    # Push encrypted backup to final backup destination
+    logecho "${RHOST}: Moving $(basename "${DEST}") to ${DEST_FINAL}"
+    cp "${DEST}".tar.gpg "${DEST_FINAL}/"
+    rm "${DEST}".tar.gpg
+
+  done # End of loop through all backup sources
+
+  # Delete all old directories except the $MAXBAK most recent
+  if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
+    oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
+    logecho "${RHOST}: Removing older backup directories: ${oldbackups}"
+    ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
+  fi
 
 done < "$HOSTS"