From 8828e1235c76ccc74b9352c95e17a32a02f421a6 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 11:39:26 +0100
Subject: [PATCH 01/10] extend docu on how backup works

---
 README.md | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/README.md b/README.md
index 9538a30..2ef1323 100644
--- a/README.md
+++ b/README.md
@@ -63,6 +63,21 @@ shortcuts:
 You can give multiple locations that shall be backed up. Just separate them by
 `|` characters. See the example file for more.
 
+## Process
+
+The script runs the following most important steps:
+1. For each host in `hosts.csv`
+    1. Check SSH connection
+    1. Compose SSH host settings
+    1. For each backup source
+        1. Resolve special backup sources
+        1. Create backup destination
+        1. rsync source to destination
+        1. tar the destination
+        1. gpg-encrypt the destination
+        1. Delete older backups
+1. Output completion info
+
 ## Manual run
 
 You can run `ssh-checker.sh` and `uberspace-backup.sh` manually. Without any arguments given, both will check/backup all hosts.
-- 
2.36.6


From f93d26b9c19cac038508d0b05108ebd9e1ffeed6 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 11:39:45 +0100
Subject: [PATCH 02/10] make a two-staged backup process, first downloading to
 a local dir

---
 uberspace-backup.sh | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index baeb2bb..f22bd4f 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -41,8 +41,8 @@ function pdate {
 }
 function logecho {
   # Echo string and copy it to log while attaching the current date
-  echo "$(pdate) $@"
-  echo "$(pdate) $@" >> "$LOG"
+  echo "$(pdate) $*"
+  echo "$(pdate) $*" >> "$LOG"
 }
 
 while read -r line; do
@@ -82,25 +82,37 @@ while read -r line; do
   NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l)
   NORDIR=$(($NORDIR + 1))
 
+  # Loop through all backup sources
   for ((i = 1; i <= $NORDIR; i++)); do
     RDIR=$(echo "$ALLRDIR" | cut -d"|" -f${i} | trim)
 
+    # Set a relative destination directory
     if [ "${RDIR}" == "%virtual" ]; then
       RDIR=/var/www/virtual/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/virtual"
+      DEST_REL="$RHOST/$DATE/virtual"
     elif [ "${RDIR}" == "%mysql" ]; then
       RDIR=mysql
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     elif [ "${RDIR}" == "%mails" ]; then
       RDIR=/home/${RUSER}/users
-      DEST="$BACKUPDIR/$RHOST/$DATE/mails"
+      DEST_REL="$RHOST/$DATE/mails"
     elif [ "${RDIR}" == "%home" ]; then
       RDIR=/home/${RUSER}
-      DEST="$BACKUPDIR/$RHOST/$DATE/home"
+      DEST_REL="$RHOST/$DATE/home"
     else
-      DEST="$BACKUPDIR/$RHOST/$DATE/$(basename "${RDIR}")"
+      DEST_REL="$RHOST/$DATE/$(basename "${RDIR}")"
     fi
 
+    # Define absolute temporary and final backup destination paths
+    # Example:
+    #   DEST=/tmp/uberspace-backup/user@example.com/2019-01-01/virtual
+    #   DEST_FINAL=/media/Uberspace/user@example.com/2019-01-01/
+    DEST="${TEMPDIR}/${DEST}"
+    DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
+
+    logecho "DEBUG: $DEST"
+    logecho "DEBUG: $DEST_FINAL"
+
     # Set Source directory, and make exception for %mysql
     SOURCE="${RDIR}"
     if [ "${RDIR}" == "mysql" ]; then
@@ -111,8 +123,9 @@ while read -r line; do
       fi
     fi
 
-    # Create backup destination if necessary
+    # Create temporary and final backup destination if necessary
     if [ ! -e "${DEST}" ]; then mkdir -p "${DEST}"; fi
+    if [ ! -e "${DEST_FINAL}" ]; then mkdir -p "${DEST_FINAL}"; fi
 
     # RSYNC
     logecho "${RHOST}: Downloading ${SOURCE} to ${DEST}"
@@ -128,9 +141,14 @@ while read -r line; do
     gpg --output "${DEST}".tar.gpg --encrypt --recipient ${GPG} "${DEST}".tar
     rm "${DEST}".tar
 
+    # Push encrypted backup to final backup destination
+    logecho "${RHOST}: Moving $(basename "${DEST}") to ${DEST_FINAL}"
+    mv "${DEST}".tar.gpg "${DEST_FINAL}/"
+
     # Delete all old directories except the $MAXBAK most recent
     if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-      logecho "${RHOST}: Removing older backups of $(basename "${DEST}")"
+      oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
+      logecho "${RHOST}: Removing older backups of $(basename "${DEST}"): ${oldbackups}"
       ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
     fi
   done
-- 
2.36.6


From 4dfc2214060a37f6789786117ec00a68f07a1a7b Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 11:44:08 +0100
Subject: [PATCH 03/10] fix temp dest dir

---
 uberspace-backup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index f22bd4f..8adc9e6 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -107,7 +107,7 @@ while read -r line; do
     # Example:
     #   DEST=/tmp/uberspace-backup/user@example.com/2019-01-01/virtual
     #   DEST_FINAL=/media/Uberspace/user@example.com/2019-01-01/
-    DEST="${TEMPDIR}/${DEST}"
+    DEST="${TEMPDIR}/${DEST_REL}"
     DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
 
     logecho "DEBUG: $DEST"
-- 
2.36.6


From 56fb5f8256c1da1120fd8b9c7286a1406cbca38f Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 11:49:21 +0100
Subject: [PATCH 04/10] make script fail fast

---
 uberspace-backup.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index 8adc9e6..670b395 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -9,6 +9,9 @@
 #
 ########################################################################
 
+# Fail fast on errors
+set -Eeuo pipefail
+
 # Set correct UTF-8 encoding (for FreeBSD jail)
 export LC_ALL=en_US.UTF-8
 
-- 
2.36.6


From 63f8fde8dd02af88b2ef5de7d0776ca52acdbdac Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 11:56:18 +0100
Subject: [PATCH 05/10] fix unbound variable error

---
 uberspace-backup.sh | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index 670b395..653ea85 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -29,8 +29,6 @@ else
   SSH_KEY=~/.ssh/id_rsa
 fi
 
-ARG1="$1"
-
 # Get current date
 DATE=$(date +"%Y-%m-%d_%H-%M")
 LOG="$CURDIR"/backup.log
@@ -54,8 +52,8 @@ while read -r line; do
 
   RHOST=$(echo "$line" | cut -d";" -f1 | trim)
 
-  # Jump to next line if this line's host does not match host of ARG1 (if given)
-  if [[ "${ARG1}" != "" ]] && [[ "${ARG1}" != "${RHOST}" ]]; then
+  # Jump to next line if this line's host does not match host of first argument (if given)
+  if [[ "${1-}" != "" ]] && [[ "${1-}" != "${RHOST}" ]]; then
     continue
   fi
   # Task ssh-checker.sh to check this host
-- 
2.36.6


From 29aa4352fd29a4a595722b3986a486bf5a3bea36 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 12:03:37 +0100
Subject: [PATCH 06/10] allow pipe to fail

---
 uberspace-backup.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index 653ea85..cef315e 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -80,7 +80,7 @@ while read -r line; do
 
   logecho "${RHOST}: Starting backups"
 
-  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l)
+  NORDIR=$(echo "$ALLRDIR" | grep -o "|" | wc -l || true)
   NORDIR=$(($NORDIR + 1))
 
   # Loop through all backup sources
-- 
2.36.6


From 9b3608d0faa9513a453efc3ca595eb021291ce39 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 12:07:35 +0100
Subject: [PATCH 07/10] use cp and rm instead of mv

---
 uberspace-backup.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index cef315e..dd4abaa 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -144,7 +144,8 @@ while read -r line; do
 
     # Push encrypted backup to final backup destination
     logecho "${RHOST}: Moving $(basename "${DEST}") to ${DEST_FINAL}"
-    mv "${DEST}".tar.gpg "${DEST_FINAL}/"
+    cp "${DEST}".tar.gpg "${DEST_FINAL}/"
+    rm "${DEST}".tar.gpg
 
     # Delete all old directories except the $MAXBAK most recent
     if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-- 
2.36.6


From d24ff0a36486eeec70c99c13ab2cedb8a57cd25a Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 12:11:31 +0100
Subject: [PATCH 08/10] reduce debug and empty lines output

---
 ssh-checker.sh      | 2 --
 uberspace-backup.sh | 3 ---
 2 files changed, 5 deletions(-)

diff --git a/ssh-checker.sh b/ssh-checker.sh
index 8c7e3e8..90ab738 100755
--- a/ssh-checker.sh
+++ b/ssh-checker.sh
@@ -68,6 +68,4 @@ while read -r line; do
     echo "[SUCCESS] SSH login possible for ${RHOST}."
   fi
 
-  echo
-
 done < "$HOSTS"
diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index dd4abaa..1f07a84 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -111,9 +111,6 @@ while read -r line; do
     DEST="${TEMPDIR}/${DEST_REL}"
     DEST_FINAL="$(dirname "${BACKUPDIR}/${DEST_REL}")"
 
-    logecho "DEBUG: $DEST"
-    logecho "DEBUG: $DEST_FINAL"
-
     # Set Source directory, and make exception for %mysql
     SOURCE="${RDIR}"
     if [ "${RDIR}" == "mysql" ]; then
-- 
2.36.6


From 83fd4294d3fc8b23bf59f8b5986ee03d69c2bc32 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 12:20:00 +0100
Subject: [PATCH 09/10] delete old backup dir after backup sources have been
 completed

---
 uberspace-backup.sh | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/uberspace-backup.sh b/uberspace-backup.sh
index 1f07a84..9801763 100755
--- a/uberspace-backup.sh
+++ b/uberspace-backup.sh
@@ -144,13 +144,14 @@ while read -r line; do
     cp "${DEST}".tar.gpg "${DEST_FINAL}/"
     rm "${DEST}".tar.gpg
 
-    # Delete all old directories except the $MAXBAK most recent
-    if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
-      oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
-      logecho "${RHOST}: Removing older backups of $(basename "${DEST}"): ${oldbackups}"
-      ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
-    fi
-  done
+  done # End of loop through all backup sources
+
+  # Delete all old directories except the $MAXBAK most recent
+  if [ $(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | wc -l | tr -d ' ') -gt $MAXBAK ]; then
+    oldbackups=$(ls -tp "${BACKUPDIR}"/"${RHOST}"/ | grep '/$' | tail -n +$(($MAXBAK + 1)))
+    logecho "${RHOST}: Removing older backup directories: ${oldbackups}"
+    ls -tpd "${BACKUPDIR}"/"${RHOST}"/* | grep '/$' | tail -n +$(($MAXBAK + 1)) | xargs -0 | xargs rm -r --
+  fi
 
 done < "$HOSTS"
 
-- 
2.36.6


From 266d9d9c35a5d0b893cbc65c645a409c104755b9 Mon Sep 17 00:00:00 2001
From: Max Mehl <mail@mehl.mx>
Date: Wed, 13 Dec 2023 12:22:01 +0100
Subject: [PATCH 10/10] add config for two-staged-backup

---
 config.cfg.sample | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/config.cfg.sample b/config.cfg.sample
index 6279421..9a2baeb 100644
--- a/config.cfg.sample
+++ b/config.cfg.sample
@@ -4,10 +4,13 @@
 # File with hosts and their backup source paths
 HOSTS="$CURDIR"/hosts.csv
 
-# root dir where backups shall be saved to
-BACKUPDIR=/var/backups/uberspace
+# Temporary download destination for backups
+TEMPDIR=/tmp/uberspace-backup
 
-# GPG fingerprint of key used for encryption  
+# root dir where backups shall be saved to
+BACKUPDIR=/mnt/remotesrv/uberspace
+
+# GPG fingerprint of key used for encryption
 GPG=6775E8DDD8CEABCC83E38CEHE6334BCA29DF8192
 
 # Maximum number of backups that shall be retained (0 to disable automatic deletion)
-- 
2.36.6