general: update min version to python3.9

- bring all CI stuff up to date
- add 3.13 to CI
- fix lots of ruff warnings and generally clean up code

Author: karlicoss

.ci/run

@@ -11,6 +11,8 @@ if ! command -v sudo; then
     }
 fi
 
+# --parallel-live to show outputs while it's running
+tox_cmd='run-parallel --parallel-live'
 if [ -n "${CI-}" ]; then
     # install OS specific stuff here
     case "$OSTYPE" in
@@ -21,6 +23,8 @@ if [ -n "${CI-}" ]; then
     cygwin* | msys* | win*)
         # windows
         :
+        # ugh. parallel stuff seems super flaky under windows, some random failures, "file used by other process" and crap like that
+        tox_cmd='run'
         ;;
     *)
         # must be linux?
@@ -37,4 +41,4 @@ if ! command -v python3 &> /dev/null; then
 fi
 
 "$PY_BIN" -m pip install --user tox
-"$PY_BIN" -m tox --parallel --parallel-live "$@"
+"$PY_BIN" -m tox $tox_cmd "$@"

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         # vvv just an example of excluding stuff from matrix
         # exclude: [{platform: macos-latest, python-version: '3.6'}]
 
@@ -53,6 +53,7 @@ jobs:
     - if: matrix.platform == 'ubuntu-latest'  # no need to compute coverage for other platforms
       uses: actions/upload-artifact@v4
       with:
+        include-hidden-files: true
         name: .coverage.mypy_${{ matrix.platform }}_${{ matrix.python-version }}
         path: .coverage.mypy/
 
@@ -67,7 +68,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.10'
 
     - uses: actions/checkout@v4
       with:

conftest.py

@@ -2,6 +2,7 @@
 # without it, pytest can't discover the package root for some reason
 # also see https://github.com/karlicoss/pytest_namespace_pkgs for more
 
+import os
 import pathlib
 from typing import Optional
 
@@ -24,6 +25,10 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
     for parent in result.parents:
         if str(parent) in namespace_pkg_dirs:
             return parent
+    if os.name == 'nt':
+        # ??? for some reason on windows it is trying to call this against conftest? but not on linux/osx
+        if path.name == 'conftest.py':
+            return resolve_pkg_path_orig(path)
     raise RuntimeError("Couldn't determine path for ", path)
 _pytest.pathlib.resolve_package_path = resolve_package_path
 
@@ -34,5 +39,9 @@ def resolve_package_path(path: pathlib.Path) -> Optional[pathlib.Path]:
 # not sure what are the consequences.. maybe it wouldn't be able to run against installed packages? not sure..
 search_pypath_orig = _pytest.main.search_pypath
 def search_pypath(module_name: str) -> str:
-    return str(root_dir)
+    mpath = root_dir / module_name.replace('.', os.sep)
+    if not mpath.is_dir():
+        mpath = mpath.with_suffix('.py')
+        assert mpath.exists(), mpath  # just in case
+    return str(mpath)
 _pytest.main.search_pypath = search_pypath

mypy.ini

@@ -1,14 +1,13 @@
 [mypy]
-namespace_packages = True
 pretty = True
 show_error_context = True
-show_error_codes = True
 show_column_numbers = True
 show_error_end = True
+warn_redundant_casts = True
 warn_unused_ignores = True
 check_untyped_defs = True
-enable_error_code = possibly-undefined
 strict_equality = True
+enable_error_code = possibly-undefined
 
 # an example of suppressing
 # [mypy-my.config.repos.pdfannots.pdfannots]

pyproject.toml

@@ -4,6 +4,7 @@ dynamic = ["version"]  # version is managed by setuptools_scm
 name = "kobuddy"
 dependencies = [
 ]
+requires-python = ">= 3.9"
 
 ## these need to be set if you're planning to upload to pypi
 description = "Backup and extract data from your Kobo reader"

ruff.toml

@@ -1,4 +1,58 @@
-ignore = [
+# NOTE: version is inferred from pyproject.toml if present
+#   however, if ruff.toml is separate, pyproject isn't even parsed..
+#   see https://github.com/astral-sh/ruff/issues/10299
+target-version = "py39"
+
+lint.extend-select = [
+    "F",    # flakes rules -- default, but extend just in case
+    "E",    # pycodestyle  -- default, but extend just in case
+    "W",    # various warnings
+
+    "B",    # 'bugbear' set -- various possible bugs
+    "C4",   # flake8-comprehensions -- unnecessary list/map/dict calls
+    "COM",  # trailing commas
+    "EXE",  # various checks wrt executable files
+    "I",    # sort imports
+    "ICN",  # various import conventions
+    "FBT",  # detect use of boolean arguments
+    "FURB", # various rules
+    "PERF", # various potential performance speedups
+    "PD",   # pandas rules
+    "PIE",  # 'misc' lints
+    "PLC",  # pylint convention rules
+    "PLR",  # pylint refactor rules
+    "PLW",  # pylint warnings
+    "PT",   # pytest stuff
+    "PYI",  # various type hinting rules
+    "RET",  # early returns
+    "RUF",  # various ruff-specific rules
+    "TID",  # various imports suggestions
+    "TRY",  # various exception handling rules
+    "UP",   # detect deprecated python stdlib stuff
+    "FA",   # suggest using from __future__ import annotations
+    "PTH",  # pathlib migration
+    "ARG",  # unused argument checks
+    "A",    # builtin shadowing
+    # "EM",  # TODO hmm could be helpful to prevent duplicate err msg in traceback.. but kinda annoying
+
+    # "ALL", # uncomment this to check for new rules!
+]
+
+# Preserve types, even if a file imports `from __future__ import annotations`
+# we need this for cachew to work with HPI types on 3.9
+# can probably remove after 3.10?
+lint.pyupgrade.keep-runtime-typing = true
+
+lint.ignore = [
+    "D",     # annoying nags about docstrings
+    "N",     # pep naming
+    "TCH",   # type checking rules, mostly just suggests moving imports under TYPE_CHECKING
+    "S",     # bandit (security checks) -- tends to be not very useful, lots of nitpicks
+    "DTZ",   # datetimes checks -- complaining about missing tz and mostly false positives
+    "FIX",   # complains about fixmes/todos -- annoying
+    "TD",    # complains about todo formatting -- too annoying
+    "ANN",   # missing type annotations? seems way to strict though
+
 ### too opinionated style checks
     "E501",  # too long lines
     "E702",  # Multiple statements on one line (semicolon)
@@ -17,9 +71,81 @@ ignore = [
     "E402",  # Module level import not at top of file
 
 ### maybe consider these soon
-# sometimes it's useful to give a variable a name even if we don't use it as a documentation
-# on the other hand, often is a sign of error
+    # sometimes it's useful to give a variable a name even if we don't use it as a documentation
+    # on the other hand, often is a sign of error
     "F841",  # Local variable `count` is assigned to but never used
-    "F401",  # imported but unused
 ###
+
+    "RUF100",  # unused noqa -- handle later
+    "RUF012",  # mutable class attrs should be annotated with ClassVar... ugh pretty annoying for user configs
+
+### these are just nitpicky, we usually know better
+    "PLR0911",  # too many return statements
+    "PLR0912",  # too many branches
+    "PLR0913",  # too many function arguments
+    "PLR0915",  # too many statements
+    "PLR1714",  # consider merging multiple comparisons
+    "PLR2044",  # line with empty comment
+    "PLR5501",  # use elif instead of else if
+    "PLR2004",  # magic value in comparison -- super annoying in tests
+###
+    "PLR0402",  # import X.Y as Y -- TODO maybe consider enabling it, but double check
+
+    "B009",  # calling gettattr with constant attribute -- this is useful to convince mypy
+    "B010",  # same as above, but setattr
+    "B011",  # complains about assert False
+    "B017",  # pytest.raises(Exception)
+    "B023",  # seems to result in false positives?
+    "B028",  # suggest using explicit stacklevel? TODO double check later, but not sure it's useful
+
+    # complains about useless pass, but has sort of a false positive if the function has a docstring?
+    # this is common for click entrypoints (e.g. in __main__), so disable
+    "PIE790",
+
+    # a bit too annoying, offers to convert for loops to list comprehension
+    # , which may heart readability
+    "PERF401",
+
+    # suggests no using exception in for loops
+    # we do use this technique a lot, plus in 3.11 happy path exception handling is "zero-cost"
+    "PERF203",
+
+    "RET504", # unnecessary assignment before returning -- that can be useful for readability
+    "RET505", # unnecessary else after return -- can hurt readability
+
+    "PLW0603",  # global variable update.. we usually know why we are doing this
+    "PLW2901",  # for loop variable overwritten, usually this is intentional
+
+    "PT004",  # deprecated rule, will be removed later
+    "PT011",  # pytest raises should is too broad
+    "PT012",  # pytest raises should contain a single statement
+
+    "COM812",  # trailing comma missing -- mostly just being annoying with long multiline strings
+
+    "PD901",   # generic variable name df
+
+    "TRY003",  # suggests defining exception messages in exception class -- kinda annoying
+    "TRY004",  # prefer TypeError -- don't see the point
+    "TRY201",  # raise without specifying exception name -- sometimes hurts readability
+    "TRY400",  # TODO double check this, might be useful
+    "TRY401",  # redundant exception in logging.exception call? TODO double check, might result in excessive logging
+
+    "PGH",  # TODO force error code in mypy instead
+
+    "TID252",  # Prefer absolute imports over relative imports from parent modules
+
+    "UP038",  # suggests using | (union) in isisntance checks.. but it results in slower code
+
+    ## too annoying
+    "T20",     # just complains about prints and pprints
+    "Q",       # flake quotes, too annoying
+    "C90",     # some complexity checking
+    "G004",    # logging statement uses f string
+    "ERA001",  # commented out code
+    "SLF001",  # private member accessed
+    "BLE001",  # do not catch 'blind' Exception
+    "INP001",  # complains about implicit namespace packages
+    "SIM",     # some if statements crap
+    "RSE102",  # complains about missing parens in exceptions
+    ##
 ]