make a two-staged backup process, first downloading to a local dir

README.md

@@ -63,6 +63,21 @@ shortcuts:
 You can give multiple locations that shall be backed up. Just separate them by
 `|` characters. See the example file for more.
 
+## Process
+
+The script runs the following most important steps:
+1. For each host in `hosts.csv`
+    1. Check SSH connection
+    1. Compose SSH host settings
+    1. For each backup source
+        1. Resolve special backup sources
+        1. Create backup destination
+        1. rsync source to destination
+        1. tar the destination
+        1. gpg-encrypt the destination
+        1. Delete older backups
+1. Output completion info
+
 ## Manual run
 
 You can run `ssh-checker.sh` and `uberspace-backup.sh` manually. Without any arguments given, both will check/backup all hosts.

uberspace-backup.sh

@@ -41,8 +41,8 @@ function pdate {
 }
 function logecho {
   # Echo string and copy it to log while attaching the current date
-  echo "$(pdate) $@"
-  echo "$(pdate) $@" >> "$LOG"
+  echo "$(pdate) $*"
+  echo "$(pdate) $*" >> "$LOG"
 }
 
 while read -r line; do
@@ -82,25 +82,37 @@ while read -r line; do
   NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l)
   NORDIR=$(($NORDIR + 1))
 
+  # Loop through all backup sources
   for ((i = 1; i <= $NORDIR; i++)); do
     RDIR=$(echo "$ALLRDIR" | cut -d"|" -f${i} | trim)
 
+    # Set a relative destination directory
     if [ "${RDIR}" == "%virtual" ]; then
       RDIR=/var/www/virtual/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/virtual"
+      DEST_REL="$RHOST/$DATE/virtual"
     elif [ "${RDIR}" == "%mysql" ]; then
       RDIR=mysql
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     elif [ "${RDIR}" == "%mails" ]; then
       RDIR=/home/${RUSER}/users
-      DEST="$BACKUPDIR/$RHOST/$DATE/mails"
+      DEST_REL="$RHOST/$DATE/mails"
     elif [ "${RDIR}" == "%home" ]; then
       RDIR=/home/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/home"
+      DEST_REL="$RHOST/$DATE/home"
     else
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     fi
 
+    # Define absolute temporary and final backup destination paths
+    # Example:
+    #   DEST=/tmp/uberspace-backup/user@example.com/2019-01-01/virtual
+    #   DEST_FINAL=/media/Uberspace/user@example.com/2019-01-01/
+    DEST="${TEMPDIR}/${DEST}"
+    DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
+
+    logecho "DEBUG: $DEST"
+    logecho "DEBUG: $DEST_FINAL"
+
     # Set Source directory, and make exception for %mysql
     SOURCE="${RDIR}"
     if [ "${RDIR}" == "mysql" ]; then
@@ -111,8 +123,9 @@ while read -r line; do
       fi
     fi
 
-    # Create backup destination if necessary
+    # Create temporary and final backup destination if necessary
     if [ ! -e "${DEST}" ]; then mkdir -p "${DEST}"; fi
+    if [ ! -e "${DEST_FINAL}" ]; then mkdir -p "${DEST_FINAL}"; fi
 
     # RSYNC
     logecho "${RHOST}: Downloading ${SOURCE} to ${DEST}"
@@ -128,9 +141,14 @@ while read -r line; do
     gpg --output "${DEST}".tar.gpg --encrypt --recipient ${GPG} "${DEST}".tar
     rm "${DEST}".tar
 
+    # Push encrypted backup to final backup destination
+    logecho "${RHOST}: Moving $(basename "${DEST}") to ${DEST_FINAL}"
+    mv "${DEST}".tar.gpg "${DEST_FINAL}/"
+
     # Delete all old directories except the $MAXBAK most recent
     if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-      logecho "${RHOST}: Removing older backups of $(basename "${DEST}")"
+      oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
+      logecho "${RHOST}: Removing older backups of $(basename "${DEST}"): ${oldbackups}"
       ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
     fi
   done