From 2c6cf362b62f6430702c37f2b335337edfd72480 Mon Sep 17 00:00:00 2001 From: mxmehl Date: Mon, 22 Jun 2015 13:02:12 +0300 Subject: [PATCH] roughly calculate remaining time, more and better help, small fixes --- split-dl.sh | 131 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 93 insertions(+), 38 deletions(-) diff --git a/split-dl.sh b/split-dl.sh index 83c6700..99c2cfe 100755 --- a/split-dl.sh +++ b/split-dl.sh @@ -6,23 +6,39 @@ CHECKSUM="md5sum" # Define application to create hashsums FILE="" MODE="" +function help { + echo "This program is used to split large files into several small +files to avoid data loss and the need to re-download them from +slow or unstable internet connections." + echo + echo "Written by Max Mehl in 2015." + echo + echo "Necessary arguments to start this application:" + echo "-m Desired MODE. Possible values are \"server\" or \"client\"" + echo " \"server\" splits the file on the server and needs a file for -f." + echo " \"client\" downloads the files and needs a download link for -f." + echo "-f File in the same directory to be splitted or link to be downloaded." + echo + echo "Optional arguments:" + echo "-s Define the desired size of splitted files. Default is $SPLITSIZE" + echo "-i Define the desired name of the info document. + Default is $INFO" + echo "-c Define the program which should be used to calculate the hashsum. + Default is $CHECKSUM" + echo + echo "Examples:" + echo "Split Hugefile.iso in parts of 50MB and use sha1sum as hashing program." + echo " split-dl -m server -f Hugefile.iso -s 50M -c sha1sum" + echo + echo "Download the splitted files of Hugefile.iso. +The hashing program has to be the same as on the server side." + echo " split-dl -m client -f http://example.com/dl-Hugefile.iso -c sha1sum" +} + while getopts h:s:i:c:f:m: opt; do case $opt in h) - echo "This program is used to split large files into several small -files to avoid data loss and the need to re-download them from -slow or unstable internet connections." - echo - echo "Necessary arguments to start this application:" - echo "-m Desired MODE. Possible values are \"server\" or \"client\"" - echo " \"server\" splits the file on the server and needs a file for -f." - echo " \"client\" downloads the files and needs a download link for -f." - echo "-f File in the same directory to be splitted or link to be downloaded." - echo - echo "Optional arguments:" - echo "-s Define the desired size of splitted files. Default is $SPLITSIZE" - echo "-i Define the desired name of the info document. Default is $INFO" - echo "-c Define the program which should be used to calculate the hashsum. Default is $CHECKSUM" + help exit 0 ;; s) @@ -46,15 +62,7 @@ done # DO NOT EDIT BELOW HERE # ---------------------- -if [ "$FILE" == "" ] || [ "$MODE" == "" ]; then - echo "Missing arguments! Please define the mode with -m and the file/link with -f. More info with -h." - exit 1 -#else - #echo "$MODE $SPLITSIZE $INFO $FILE $CHECKSUM" -fi - - -# # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # FUNCTIONS # # # # function checkwait { @@ -76,7 +84,14 @@ function comphash { } function getdu { - du -b $1 | awk -F" " '{ print $1 }' + DU=$(du -b $1 | awk -F" " '{ print $1 }') + if [ -d $1 ] && [ "$BLOCK" != "" ]; then + # Reduce the size of an directory by the block size + # $BLOCK is defined in the one-time-run section of this script + echo $(($DU - $BLOCK)) + else + echo $DU + fi } function compdu { @@ -122,8 +137,27 @@ function checkfolder { fi } - +# # # # # # # # # # # # # # # # # # # # # # # # # +# ONE-TIME-ACTIONS # # # # + +# Check if -m and -f are given +if [ "$FILE" == "" ] || [ "$MODE" == "" ]; then + echo "Missing arguments! Please define the mode with -m and the file/link with -f." + echo + help + exit 1 +#else + #echo "$MODE $SPLITSIZE $INFO $FILE $CHECKSUM" +fi + +# Check size in bytes of an empty directory +TMP=$(mktemp -d) +BLOCK=$(getdu $TMP) +rm -r $TMP + + +# # # # # # # # # # # # # # # # # # # # # # # # # # SERVER MODE # # # # if [ $MODE == "server" ]; then @@ -167,7 +201,7 @@ checkfolder $FOLDER INFONAME=$INFO INFO="$FOLDER/$INFONAME" -echo "[INFO] Calculating size and HASH sum..." +echo "[INFO] Calculating size and hashsum..." BIGNAME=$(basename $FILE) BIGSIZE=$(getdu $FILE) BIGHASH=$(gethash $FILE) @@ -176,6 +210,11 @@ echo "BIGNAME=$BIGNAME" >> $INFO echo "BIGSIZE=$BIGSIZE" >> $INFO echo "BIGHASH=$BIGHASH" >> $INFO echo >> $INFO +# Variables info +echo "CHECKSUM=$CHECKSUM" >> $INFO +echo "SPLITSIZE=$SPLITSIZE" >> $INFO +echo "INFO=$INFO" >> $INFO +echo >> $INFO echo "[INFO] Splitting big file into smaller parts..." split --verbose -a 4 -b $SPLITSIZE $FILE $FOLDER/dl- @@ -204,7 +243,7 @@ echo "[INFO] Please move the folder \"$FOLDER\" to a web-accessible directory an fi # /SERVER MODE -# # # # +# # # # # # # # # # # # # # # # # # # # # # # # # # CLIENT MODE # # # # if [ $MODE == "client" ]; then @@ -223,7 +262,7 @@ wget -q $URL/$INFO # Rename info.cfg to avoid colissions and source it source $INFO mv $INFO $BIGNAME-$INFO -INFO=$BIGNAME-$INFO +INFO=dl-$BIGNAME-$INFO FOLDER="dl-$BIGNAME" @@ -239,6 +278,7 @@ echo checkfolder $FOLDER # Checks and creates folder +# Downloading every single small file until [ "$STATUS" == "F" ]; do for ((i = 0; i < ${#SMALLNAME[*]}; i++)); do SMALLNAME=${SMALLNAME[$i]} @@ -255,7 +295,10 @@ until [ "$STATUS" == "F" ]; do echo "This is file $NO of total $SMALLNO files." echo "Already downloaded "$(getdu $FOLDER)" of $BIGSIZE bytes." echo - + + # Define starttime + TSTART=$(date +"%s") + # File doesn't exist yet, so start fresh download if [ ! -e $SMALLPATH ]; then echo "[INFO] File doesn't exist yet. Starting new download." @@ -282,9 +325,9 @@ until [ "$STATUS" == "F" ]; do break fi - # File already exists, has correct size, but has wrong HASH + # File already exists, has correct size, but has wrong hashsum elif [ -e $SMALLPATH ] && $(compdu $SMALLPATH SMALLSIZE) && ! $(comphash $SMALLPATH $SMALLHASH); then - echo "[ERROR] HASH is different but file has same size." + echo "[ERROR] Hashsum is different but file has same size." echo "[INFO] Deleting file and starting new download." checkwait rm $SMALLPATH @@ -298,7 +341,7 @@ until [ "$STATUS" == "F" ]; do break fi - # File already exists, has correct size, and has correct HASH + # File already exists, has correct size, and has correct hashsum elif [ -e $SMALLPATH ] && $(compdu $SMALLPATH $SMALLSIZE) && $(comphash $SMALLPATH $SMALLHASH); then echo "[SUCCESS] File already exists and is valid." STATUS="C" @@ -309,7 +352,19 @@ until [ "$STATUS" == "F" ]; do STATUS="E" fi - # Check if download is finished + # Define time when the download finished + echo + TEND=$(date +"%s") + TDIFF=$[$TEND-$TSTART] + if [ $TDIFF -ge "3" ]; then + # Calc bytes per second + BPS=$[$SMALLSIZE / $TDIFF] + DUDIFF=$[$BIGSIZE - $(getdu $FOLDER)] + TREST=$[$DUDIFF / $BPS] + echo "[INFO] The last file was downloaded at ~$[$BPS / 1024] KB/s. ETA $(($TREST / 60)):$(($TREST % 60))." + fi + + # Check if all small downloads are finished if [ $NO -lt $SMALLNO ]; then echo "[INFO] Starting next download." STATUS="C" @@ -341,9 +396,9 @@ if [ $YN == "y" ]; then fi fi -# Check big file for HASH and size +# Check big file for hashsum and size sleep 2 # In rare cases, this can prevent a wrong du size -echo "[INFO] Checking correct size and HASH hashsum for rebuilt big file..." +echo "[INFO] Checking correct size and hashsum for rebuilt big file..." # Compare sizes if $(compdu $BIGNAME $BIGSIZE); then @@ -352,11 +407,11 @@ else echo "[ERROR] The size of the completed file is incorrect." fi -# Compare HASH sum +# Compare hashsum if $(comphash $BIGNAME $BIGHASH); then - echo "[SUCCESS] The HASH sum of the completed file is corrent." + echo "[SUCCESS] The checksum of the completed file is corrent." else - echo "[ERROR] The HASH sum of the completed file is incorrect." + echo "[ERROR] The sum of the completed file is incorrect." fi