initial commit
This commit is contained in:
199
seafile_mirror.py
Executable file
199
seafile_mirror.py
Executable file
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Handle clean read-only (re-)syncs of Seafile libraries to mirror them"""
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from time import sleep
|
||||
|
||||
import yaml
|
||||
|
||||
from functions.cachedb import db_read
|
||||
from functions.helpers import findstring, get_lock
|
||||
from functions.seafile import (
|
||||
sf_bump_cache_status,
|
||||
sf_desync_all,
|
||||
sf_lastsync_old_enough,
|
||||
sf_runcmd,
|
||||
sf_waitforsynced,
|
||||
)
|
||||
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("-c", "--configdir", required=True, help="The config directory")
|
||||
parser.add_argument(
|
||||
"-d",
|
||||
"--dry",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Do not modify anything. Useful for being informed about which "
|
||||
"libraries are due to be synced",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-f",
|
||||
"--force",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Force re-sync of libraries even if they are newer than the configured limit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v",
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Print and log DEBUG messages",
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function"""
|
||||
# Logging
|
||||
log = logging.getLogger()
|
||||
logging.basicConfig(
|
||||
encoding="utf-8",
|
||||
format="[%(asctime)s] %(levelname)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
# Log to file and stdout
|
||||
handlers=[
|
||||
logging.FileHandler(logfile),
|
||||
logging.StreamHandler(),
|
||||
],
|
||||
)
|
||||
# Set loglevel based on --verbose flag
|
||||
if args.verbose:
|
||||
log.setLevel(logging.DEBUG)
|
||||
else:
|
||||
log.setLevel(logging.INFO)
|
||||
|
||||
# Get lock for this process
|
||||
get_lock("seafile_backup")
|
||||
|
||||
# Read configfile
|
||||
with open(configfile, "r", encoding="UTF-8") as yamlfile:
|
||||
config = yaml.safe_load(yamlfile)
|
||||
|
||||
# Populate cache dictionary
|
||||
cache = db_read(cachefile)
|
||||
|
||||
# Check if there are still libraries in `list` or `status`. Desync them if
|
||||
# possible. Do not run in dry-run
|
||||
if not args.dry:
|
||||
sf_desync_all(cache)
|
||||
|
||||
# Create list of libraries we handle(d) for final output
|
||||
libsdone = []
|
||||
|
||||
# Go through users in config
|
||||
for access in config:
|
||||
# Setting variables for this server/user/pass combination
|
||||
server = access["server"]
|
||||
user = access["user"]
|
||||
password = access["password"]
|
||||
resyncinterval = access["resync_interval_days"]
|
||||
authlist = [server, user, password]
|
||||
|
||||
logging.info(
|
||||
"Checking all libraries for user %s on server %s for "
|
||||
"whether they are due for a re-sync",
|
||||
user,
|
||||
server,
|
||||
)
|
||||
|
||||
# Get remotely available libraries
|
||||
remotelibs = sf_runcmd(authlist, "list-remote")
|
||||
|
||||
for lib in access["libs"]:
|
||||
# Setting variables for this library
|
||||
libdir = Path(lib["dir"])
|
||||
libname = lib["name"]
|
||||
libid = lib["id"]
|
||||
# Set resync interval if there is a lib-specific setting. Otherwise default
|
||||
libresyncinterval = (
|
||||
lib["resync_interval_days"]
|
||||
if "resync_interval_days" in lib
|
||||
else resyncinterval
|
||||
)
|
||||
|
||||
# Check if last sync of library is older than resync_interval_days
|
||||
if sf_lastsync_old_enough(cache, libid, args.force, libresyncinterval):
|
||||
logging.info(
|
||||
"Starting to re-sync library %s (%s) to %s", libname, libid, libdir
|
||||
)
|
||||
else:
|
||||
logging.info(
|
||||
"Local mirror of library %s (%s) at %s is still recent enough. Skipping it.",
|
||||
libname,
|
||||
libid,
|
||||
libdir,
|
||||
)
|
||||
continue
|
||||
|
||||
# Check if desired library exists remotely
|
||||
if findstring(remotelibs, libid):
|
||||
logging.debug("The library %s exists remotely. Continuing...", libname)
|
||||
else:
|
||||
# If the library does not exist remotely, we don't continue
|
||||
# Otherwise, we would delete data which cannot be retrieved again!
|
||||
logging.warning(
|
||||
"The library %s does not exist remotely. Aborting resyncing this library.",
|
||||
libname,
|
||||
)
|
||||
# Start next iteration of loop (next library)
|
||||
continue
|
||||
|
||||
if args.dry:
|
||||
logging.info(
|
||||
"Running in dry run mode. Aborting resync of library %s which would happen now",
|
||||
libname,
|
||||
)
|
||||
continue
|
||||
|
||||
# Delete libdir if it exists
|
||||
if libdir.exists() and libdir.is_dir():
|
||||
logging.debug("Deleting library directory %s", libdir)
|
||||
shutil.rmtree(libdir)
|
||||
else:
|
||||
logging.debug("Library directory did not exist before: %s", libdir)
|
||||
|
||||
# Re-create directory
|
||||
logging.debug("Creating library directory %s", libdir)
|
||||
Path(libdir).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Trigger sync of library
|
||||
logging.debug("Starting to sync library %s to %s", libname, libdir)
|
||||
sf_runcmd(authlist, "sync", "-l", libid, "-d", libdir)
|
||||
sf_bump_cache_status(cache, libid, status="started")
|
||||
|
||||
# Sleep a second to populate `status`
|
||||
sleep(1)
|
||||
|
||||
# Check regularly how the syncing progress is and wait for it to finish
|
||||
syncduration = sf_waitforsynced(libname)
|
||||
|
||||
# Library is synchronised, now we desync it again
|
||||
logging.debug(
|
||||
"Desyncing library %s stored at %s after it has been synced",
|
||||
libname,
|
||||
libdir,
|
||||
)
|
||||
sf_runcmd(None, "desync", "-d", libdir)
|
||||
|
||||
# Update libsdone and cache
|
||||
libsdone.append(libname)
|
||||
sf_bump_cache_status(cache, libid, status="synced", duration=syncduration)
|
||||
|
||||
logging.info(
|
||||
"Library %s (%s) has been re-synced to %s", libname, libid, libdir
|
||||
)
|
||||
|
||||
logging.info("Fully re-synced the following libraries: %s", ", ".join(libsdone))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parser.parse_args()
|
||||
# Set files depending on configdir
|
||||
configdir = args.configdir.rstrip("/") + "/"
|
||||
configfile = configdir + "seafile_mirror.conf.yaml"
|
||||
cachefile = configdir + ".seafile_mirror.db.json"
|
||||
logfile = configdir + "seafile_mirror.log"
|
||||
main()
|
||||
Reference in New Issue
Block a user