Merge pull request 'chore(deps): update dependency mypy to v1.20.1' (#72) from renovate/all-dev-dependencies into main

Reviewed-on: #64

.gitea/actions/poetrybuild/action.yaml

@@ -0,0 +1,21 @@
+# SPDX-FileCopyrightText: 2023 DB Systel GmbH
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: "Reusable Poetry build workflow"
+inputs:
+  poetry_install_args:
+    default: ""
+    description: "Value for additional poetry install arguments"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Add pipx to PATH
+      run: echo "/root/.local/bin" >> ${GITHUB_PATH}
+    - name: Install poetry
+      run: pipx install poetry
+    - name: Install poetry package
+      run: poetry install --no-interaction ${{ inputs.poetry_install_args }}
+      shell: bash

.gitea/workflows/linting.yaml

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Python Linters
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  pylint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install dependencies
+        uses: ./.gitea/actions/poetrybuild
+      - name: Lint with pylint
+        run: poetry run pylint seafile_mirror
+
+  formatting:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install dependencies
+        uses: ./.gitea/actions/poetrybuild
+      - name: Test formatting with isort and black
+        run: |
+          poetry run isort --check seafile_mirror/
+          poetry run black .
+
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install dependencies
+        uses: ./.gitea/actions/poetrybuild
+      - name: Test typing with mypy
+        run: poetry run mypy

.gitea/workflows/publish.yaml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Python package
+on:
+  push:
+    tags:
+      - "v*.*.*"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Build and publish to PyPI
+        uses: https://github.com/JRubics/poetry-publish@v2.1
+        with:
+          pypi_token: ${{ secrets.PYPI_TOKEN }}

.gitea/workflows/reuse.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: REUSE Compliance
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  reuse:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Check REUSE Compliance
+        uses: https://github.com/fsfe/reuse-action@v6

.gitea/workflows/selftests.yaml

@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: 2023 DB Systel GmbH
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: Selftests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  # Test building the package and installing it via pip3
+  test-build-install:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v6
+      - name: Add pipx to PATH
+        run: echo "/root/.local/bin" >> ${GITHUB_PATH}
+      - name: Install poetry
+        run: pipx install poetry
+      - name: Build package
+        run: poetry build
+      - name: Install package
+        run: pip3 install dist/seafile_mirror-*.tar.gz
+      - name: Run package
+        run: |
+          seafile-mirror --version
+          seafile-mirror --help
+
+  # Run tool and sync a test library
+  test-sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install dependencies
+        uses: ./.gitea/actions/poetrybuild
+        # using minimal set of dependencies
+        with:
+          poetry_install_args: "--without dev"
+      - name: Install seaf-cli
+        run: |
+          apt-get update
+          apt-get install -y seafile-cli
+      - name: Configure seaf-cli and start daemon
+        run: |
+          seaf-cli init -d /tmp
+          seaf-cli start
+      - name: Fill configuration for test library
+        run: |
+          sed -i "s|__seafile_server__|${{ secrets.SEAFILE_SERVER }}|" tests/seafile_mirror.conf.yaml
+          sed -i "s|__seafile_user__|${{ secrets.SEAFILE_USER }}|" tests/seafile_mirror.conf.yaml
+          sed -i "s|__seafile_pass__|${{ secrets.SEAFILE_PASS }}|" tests/seafile_mirror.conf.yaml
+      - name: Sync the library
+        run: poetry run seafile-mirror -c tests -v
+      - name: Attempt to find expected string in library
+        run: grep "__ci_test_expect__" tests/ci-test/testfile.txt

.gitignore

@@ -2,6 +2,10 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+dist/
+venv/
 .seafile_mirror.db.json
+/seafile_mirror.conf.yaml
 __pycache__/
 *.log
+tests/ci-test/

README.md

@@ -6,6 +6,10 @@ SPDX-License-Identifier: Apache-2.0
 
 # Seafile Mirror
 
+[![REUSE status](https://api.reuse.software/badge/src.mehl.mx/mxmehl/seafile-mirror)](https://api.reuse.software/info/src.mehl.mx/mxmehl/seafile-mirror)
+[![The latest version of reuse can be found on PyPI.](https://img.shields.io/pypi/v/seafile-mirror.svg)](https://pypi.org/project/seafile-mirror/)
+[![Information on what versions of Python the tool supports can be found on PyPI.](https://img.shields.io/pypi/pyversions/seafile-mirror.svg)](https://pypi.org/project/seafile-mirror/)
+
 A Python tool to handle clean read-only (re-)syncs of
 [Seafile](https://www.seafile.com) libraries with the intention to mirror them.
 
@@ -39,30 +43,47 @@ Seafile servers!
 ## Install
 
 The tool depends on the following applications:
-* `Python 3` and its library `yaml`
-* [`seafile-cli`](https://help.seafile.com/syncing_client/linux-cli/), available e.g. in [Debian](https://packages.debian.org/bullseye/seafile-cli)
+* `Python 3`
+* [`seafile-cli`](https://help.seafile.com/syncing_client/linux-cli/), available
+  e.g. in [Debian](https://packages.debian.org/bullseye/seafile-cli)
 
-You can execute the tool with `python3 seafile_mirror.py`. The `--help` flag
-informs you about the required and available commands.
+You can install the latest release via `pip3 install seafile-mirror`.
+
+The tool is executable by `seafile-mirror`. The `--help` flag informs you about
+the required and available commands.
 
 There is also an [Ansible
 role](https://src.mehl.mx/mxmehl/seafile-mirror-ansible) that takes care of
-installing the tool, setting up a systemd service, and running it daily.
+installing the tool via `pipx`, setting up a systemd service, and running it
+daily.
 
-To keep the Seafile daemon that is required for `seafile-cli` running in the background, check out this [exemplary systemd service](examples/seaf-daemon.service).
+To keep the Seafile daemon that is required for `seafile-cli` running in the
+background, check out this [exemplary systemd
+service](examples/seaf-daemon.service).
 
 ## Configuration
 
-Configuration is done in a YAML file called `seafile_mirror.conf.yaml`. You can find an example [here](examples/seafile_mirror.conf.yaml).
+Configuration is done in a YAML file called `seafile_mirror.conf.yaml`. You can
+find an example [here](examples/seafile_mirror.conf.yaml).
 
-If that configuration file resides in the same location as the `seafile_mirror.py` file you are running, you should provide `--configdir ./`.
+If that configuration file resides in the same location your current working
+directory, you should provide `--configdir ./`.
 
 ## Logging and caching
 
 The tool creates `seafile_mirror.log` in addition to the log to the standard
-output. With `-v` you can print DEBUG messages that will help you in case of
-problems.
+output in the configuration directory. With `-v` you can print DEBUG messages
+that will help you in case of problems.
 
 It also caches the current status of synced libraries and their latest full
 download in the file `.seafile_mirror.db.json`. Do not delete this file unless
 you don't mind that the tool will re-sync all libraries in the next run.
+
+## Contribute and Development
+
+Contributions are welcome! The development is easiest with `poetry`: `poetry
+install` and `poetry run seafile-mirror` will get you started.
+
+## License
+
+Apache-2.0, Copyright Max Mehl

functions/__init__.py

@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
-#
-# SPDX-License-Identifier: Apache-2.0

poetry.lock.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+
+SPDX-License-Identifier: Apache-2.0

pyproject.toml

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[tool.poetry]
+name = "seafile-mirror"
+version = "0.2.2"
+description = "Handle clean read-only (re-)syncs of Seafile libraries to mirror them"
+authors = ["Max Mehl <mail@mehl.mx>"]
+license = "Apache-2.0"
+readme = "README.md"
+repository = "https://src.mehl.mx/mxmehl/seafile-mirror"
+packages = [{ include = "seafile_mirror" }]
+
+[tool.poetry.scripts]
+seafile-mirror = 'seafile_mirror.seafile_mirror:main'
+
+[tool.poetry.dependencies]
+python = "^3.10"
+pyyaml = "^6.0.1"
+
+[tool.poetry.group.dev.dependencies]
+pylint = "^4.0.0"
+isort = "^8.0.0"
+black = "^26.0.0"
+mypy = "^1.5.1"
+pylama = "^8.4.1"
+types-pyyaml = "^6.0.12.11"
+bump-my-version = "^1.0.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+# FORMATTING settings
+[tool.isort]
+profile = "black"
+
+[tool.black]
+line-length = 100
+
+# MYPY settings
+[tool.mypy]
+files = ["seafile_mirror/*.py"]
+
+# Bump-My-Version
+[tool.bumpversion]
+commit = true
+tag = true
+allow_dirty = false
+tag_name = "v{new_version}"
+
+[[tool.bumpversion.files]]
+filename = "pyproject.toml"
+regex = true
+search = "^version = \"{current_version}\""
+replace = "version = \"{new_version}\""

renovate.json5

@@ -0,0 +1,18 @@
+{
+  $schema: "https://docs.renovatebot.com/renovate-schema.json",
+  extends: ["local>renovate-bot/renovate-config"],
+  "packageRules": [
+    // Group and merge all dev dependencies, even major versions
+    {
+      "matchDepTypes": ["dev"],
+      "groupName": "all-dev-dependencies",
+      "automerge": true
+    },
+    // Merge all other patch and minor dependencies
+    {
+      "matchUpdateTypes": ["minor", "patch"],
+      "matchCurrentVersion": "!/^0/",
+      "automerge": true
+    }
+  ]
+}

renovate.json5.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+
+SPDX-License-Identifier: Apache-2.0

seafile_mirror/__init__.py

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+
+"""Global init file"""
+
+from importlib.metadata import version
+
+__version__ = version("seafile-mirror")

seafile_mirror/_cachedb.py

@@ -43,9 +43,7 @@ def db_update(dbdict, libid, **kwargs):
     if libid not in dbdict:
         dbdict[libid] = {}
     for key, value in kwargs.items():
-        logging.debug(
-            "Updating '%s' of library '%s' in in-memory cache dictionary", key, libid
-        )
+        logging.debug("Updating '%s' of library '%s' in in-memory cache dictionary", key, libid)
         dbdict[libid][key] = value
 
     db_write(dbdict)

seafile_mirror/_helpers.py

@@ -39,3 +39,14 @@ def findstring(text, string):
 def countlines(string: str) -> int:
     """Count number of lines in a variable"""
     return len(string.splitlines())
+
+
+def convert_bytes(size):
+    """Convert bytes to KB, MB etc depending on size"""
+    power = 1024
+    level = 0
+    labels = {0: "B", 1: "KB", 2: "MB", 3: "GB", 4: "TB"}
+    while size > power:
+        size /= power
+        level += 1
+    return f"{round(size, 2)} {labels[level]}"

seafile_mirror/_seafile.py

@@ -10,7 +10,7 @@ import subprocess
 import sys
 from time import sleep
 
-from .cachedb import db_get_library_key, db_update
+from ._cachedb import db_get_library_key, db_update
 
 # Constants
 # Seafile CLI command
@@ -38,7 +38,7 @@ def sf_runcmd(auth: list, *arguments: str) -> str:
     # check for errors
     if ret.returncode != 0:
         logging.error("The command '%s' returned an error: %s", ret.args, ret.stderr)
-        return False
+        return ""
 
     return ret.stdout.decode("UTF-8")
 
@@ -50,20 +50,20 @@ def sf_parse(output: str, fromcommand: str) -> list:
     for lib in output.splitlines()[1:]:
         # If list, split is by " ", and structure is name,id,dir
         if fromcommand == "list":
-            lib = [x.strip() for x in lib.split(" ")]
-            lib = {"name": lib[0], "id": lib[1], "dir": lib[2]}
-            libs.append(lib)
+            lib_info = [x.strip() for x in lib.split(" ")]
+            lib_dict = {"name": lib_info[0], "id": lib_info[1], "dir": lib_info[2]}
+            libs.append(lib_dict)
         # If status, split is by "\t", and structure is name,status,progress
         elif fromcommand == "status":
             lib_tmp = [x.strip() for x in lib.split("\t")]
-            lib = {"name": lib_tmp[0], "status": lib_tmp[1]}
+            lib_dict = {"name": lib_tmp[0], "status": lib_tmp[1]}
             # Add progress if it exists
             try:
-                lib["progress"] = lib_tmp[2]
+                lib_dict["progress"] = lib_tmp[2]
             except IndexError:
-                lib["progress"] = None
+                lib_dict["progress"] = ""
 
-            libs.append(lib)
+            libs.append(lib_dict)
 
     return libs
 
@@ -71,8 +71,8 @@ def sf_parse(output: str, fromcommand: str) -> list:
 def sf_getstatus(libname: str) -> dict:
     """Return the current status of a library (name, status, progress)"""
     # Get output of `status` and parse it
-    libsstatus = sf_runcmd(None, "status")
-    libsstatus = sf_parse(libsstatus, "status")
+    libsstatus_cmd = sf_runcmd([], "status")
+    libsstatus = sf_parse(libsstatus_cmd, "status")
 
     # In the list of statuses, get the one for the requested library. None if no match
     status = next((item for item in libsstatus if item["name"] == libname), None)
@@ -140,10 +140,10 @@ def sf_desync_all(cache):
         sys.exit(1)
 
 
-def sf_waitforsynced(libname) -> int:
+def sf_waitforsynced(libname) -> float:
     """Regularly check status of the library that started to sync"""
     libsynced = False
-    syncwaitmins = 0
+    syncwaitmins: float = 0
     nostatus, nostatus_limit = 0, 10
     while libsynced is not True:
         libstatus = sf_getstatus(libname)
@@ -183,8 +183,8 @@ def sf_waitforsynced(libname) -> int:
                     libname,
                     nostatus_limit,
                 )
-                sf_runcmd(None, "stop")
-                sf_runcmd(None, "start")
+                sf_runcmd([], "stop")
+                sf_runcmd([], "start")
 
         # If library not synced yet or no status available, wait
         if not libsynced:

seafile_mirror/seafile_mirror.py

@@ -14,9 +14,10 @@ from time import sleep
 
 import yaml
 
-from functions.cachedb import db_read
-from functions.helpers import findstring, get_lock
-from functions.seafile import (
+from . import __version__
+from ._cachedb import db_read
+from ._helpers import convert_bytes, findstring, get_lock
+from ._seafile import (
     sf_bump_cache_status,
     sf_desync_all,
     sf_lastsync_old_enough,
@@ -26,6 +27,11 @@ from functions.seafile import (
 
 parser = argparse.ArgumentParser(description=__doc__)
 parser.add_argument("-c", "--configdir", required=True, help="The config directory")
+parser.add_argument(
+    "-l",
+    "--logfile",
+    help="The path to the logfile. Default: <configdir>/seafile_mirror.log",
+)
 parser.add_argument(
     "-d",
     "--dry",
@@ -48,10 +54,21 @@ parser.add_argument(
     default=False,
     help="Print and log DEBUG messages",
 )
+parser.add_argument("--version", action="version", version="%(prog)s " + __version__)
 
 
-def main():
+def main():  # pylint: disable=too-many-locals, too-many-statements, too-many-branches
     """Main function"""
+    args = parser.parse_args()
+    # Set files depending on configdir
+    configdir = args.configdir.rstrip("/") + "/"
+    configfile = configdir + "seafile_mirror.conf.yaml"
+    cachefile = configdir + ".seafile_mirror.db.json"
+    if args.logfile:
+        logfile = args.logfile
+    else:
+        logfile = configdir + "seafile_mirror.log"
+
     # Logging
     log = logging.getLogger()
     logging.basicConfig(
@@ -86,7 +103,7 @@ def main():
         sf_desync_all(cache)
 
     # Create list of libraries we handle(d) for final output
-    libsdone = []
+    libsdone = {"libs": [], "bytes": 0, "time": 0}
 
     # Go through users in config
     for access in config:
@@ -114,16 +131,12 @@ def main():
             libid = lib["id"]
             # Set resync interval if there is a lib-specific setting. Otherwise default
             libresyncinterval = (
-                lib["resync_interval_days"]
-                if "resync_interval_days" in lib
-                else resyncinterval
+                lib["resync_interval_days"] if "resync_interval_days" in lib else resyncinterval
             )
 
             # Check if last sync of library is older than resync_interval_days
             if sf_lastsync_old_enough(cache, libid, args.force, libresyncinterval):
-                logging.info(
-                    "Starting to re-sync library %s (%s) to %s", libname, libid, libdir
-                )
+                logging.info("Starting to re-sync library %s (%s) to %s", libname, libid, libdir)
             else:
                 logging.info(
                     "Local mirror of library %s (%s) at %s is still recent enough. Skipping it.",
@@ -183,22 +196,33 @@ def main():
             )
             sf_runcmd(None, "desync", "-d", libdir)
 
+            # Get size of directory (libdir) in bytes
+            # Note: this is not fully equivalent with what `du` would show. It's
+            # caused by the fact that `du` considers filesystem block sizes
+            libdirsize = sum(f.stat().st_size for f in libdir.glob("**/*") if f.is_file())
+
             # Update libsdone and cache
-            libsdone.append(libname)
+            libsdone["libs"].append(libname)
+            libsdone["bytes"] += libdirsize
+            libsdone["time"] += syncduration
             sf_bump_cache_status(cache, libid, status="synced", duration=syncduration)
 
             logging.info(
-                "Library %s (%s) has been re-synced to %s", libname, libid, libdir
+                "Library %s (%s) has been re-synced to %s. Duration: %s minutes. Size: %s",
+                libname,
+                libid,
+                libdir,
+                round(syncduration),
+                convert_bytes(libdirsize),
             )
 
-    logging.info("Fully re-synced the following libraries: %s", ", ".join(libsdone))
+    logging.info(
+        "Fully re-synced the following libraries: %s. Total duration: %s minutes. Total size: %s",
+        ", ".join(libsdone["libs"]),
+        round(libsdone["time"]),
+        convert_bytes(libsdone["bytes"]),
+    )
 
 
 if __name__ == "__main__":
-    args = parser.parse_args()
-    # Set files depending on configdir
-    configdir = args.configdir.rstrip("/") + "/"
-    configfile = configdir + "seafile_mirror.conf.yaml"
-    cachefile = configdir + ".seafile_mirror.db.json"
-    logfile = configdir + "seafile_mirror.log"
     main()

tests/seafile_mirror.conf.yaml

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: 2023 Max Mehl <https://mehl.mx>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Configuration file for seafile-mirror which will be filled in and applied in
+# the CI test
+- server: __seafile_server__
+  user: __seafile_user__
+  password: __seafile_pass__
+  # The default resync interval
+  resync_interval_days: 7
+  # Define the libraries which shall be synced
+  libs:
+  - name: ci-test
+    # ID of the Seafile library (can be seen in the web UI)
+    id: 558667f5-cee2-443d-8f8e-668e70a23e9f
+    # local directory where the mirror shall be created
+    dir: tests/ci-test