Merge pull request #69 from lsst-sqre/tickets/DM-43173

DM-43173: Update to ruff linting and pydantic 2

.github/workflows/ci.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.0
@@ -40,7 +40,7 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v4
@@ -97,7 +97,7 @@ jobs:
       - name: Run tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           tox-envs: "docs,docs-linkcheck"
 
       # Only attempt documentation uploads for long-lived branches, tagged

.github/workflows/dependencies.yaml

@@ -16,7 +16,7 @@ jobs:
       - name: Run neophile
         uses: lsst-sqre/run-neophile@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           mode: pr
           types: pre-commit
           app-id: ${{ secrets.NEOPHILE_APP_ID }}

.github/workflows/periodic-ci.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       matrix:
         python:
-          - "3.11"
+          - "3.12"
 
     steps:
       - uses: actions/checkout@v4
@@ -29,7 +29,7 @@ jobs:
       - name: Run neophile
         uses: lsst-sqre/run-neophile@v1
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           mode: update
 
       - name: Run tests in tox

.pre-commit-config.yaml

@@ -7,12 +7,7 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.278
+    rev: v0.3.0
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
-
-  - repo: https://github.com/psf/black
-    rev: 23.11.0
-    hooks:
-      - id: black

Dockerfile

@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.11.5-slim-bullseye as base-image
+FROM python:3.12.2-slim-bullseye as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .

changelog.d/20240306_111612_jsick_DM_43173.md

@@ -0,0 +1,6 @@
+### Other changes
+
+- Updated to Python 3.12
+- Updated to Pydantic 2
+- Adopted Ruff for linting and formatting, replacing black, flake8, and isort.
+- Switch to using Annotated for Pydantic models and FastAPI path functions.

pyproject.toml

@@ -10,7 +10,7 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Intended Audience :: Developers",
     "Natural Language :: English",
     "Operating System :: POSIX",
@@ -56,24 +56,6 @@ exclude_lines = [
     "if TYPE_CHECKING:"
 ]
 
-[tool.black]
-line-length = 79
-target-version = ['py311']
-exclude = '''
-/(
-    \.eggs
-  | \.git
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | build
-  | dist
-)/
-'''
-# Use single-quoted strings so TOML treats the string like a Python r-string
-# Multi-line strings are implicitly treated by black as regular expressions
-
 [tool.pytest.ini_options]
 asyncio_mode = "strict"
 python_files = [
@@ -100,6 +82,19 @@ init_typed = true
 warn_required_dynamic_aliases = true
 warn_untyped_fields = true
 
+[tool.scriv]
+categories = [
+    "Backwards-incompatible changes",
+    "New features",
+    "Bug fixes",
+    "Other changes",
+]
+entry_title_template = "{{ version }} ({{ date.strftime('%Y-%m-%d') }})"
+format = "md"
+md_header_level = "2"
+new_fragment_template = "file:changelog.d/_template.md.jinja"
+skip_fragments = "_template.md.jinja"
+
 # The rule used with Ruff configuration is to disable every lint that has
 # legitimate exceptions that are not dodgy code, rather than cluttering code
 # with noqa markers. This is therefore a reiatively relaxed configuration that
@@ -112,6 +107,9 @@ exclude = [
     "docs/**",
 ]
 line-length = 79
+target-version = "py312"
+
+[tool.ruff.lint]
 ignore = [
     "ANN101",  # self should not have a type annotation
     "ANN102",  # cls should not have a type annotation
@@ -121,94 +119,80 @@ ignore = [
     "ARG005",  # unused lambda arguments are often legitimate
     "BLE001",  # we want to catch and report Exception in background tasks
     "C414",    # nested sorted is how you sort by multiple keys with reverse
-    "COM812",  # omitting trailing commas allows black autoreformatting
     "D102",    # sometimes we use docstring inheritence
     "D104",    # don't see the point of documenting every package
     "D105",    # our style doesn't require docstrings for magic methods
     "D106",    # Pydantic uses a nested Config class that doesn't warrant docs
-    "D205",    # Summary-only docs are fine for some methods
+    "D205",    # our documentation style allows a folded first line
     "EM101",   # justification (duplicate string in traceback) is silly
     "EM102",   # justification (duplicate string in traceback) is silly
-    "FIX002",  # TODOs are okay
     "FBT003",  # positional booleans are normal for Pydantic field defaults
+    "FIX002",  # point of a TODO comment is that we're not ready to fix it
     "G004",    # forbidding logging f-strings is appealing, but not our style
     "RET505",  # disagree that omitting else always makes code more readable
+    "PLR0911", # often many returns is clearer and simpler style
     "PLR0913", # factory pattern uses constructors with many arguments
     "PLR2004", # too aggressive about magic values
-    "PD011",   # allow use of values as a method (not using numpy here)
+    "PLW0603", # yes global is discouraged but if needed, it's needed
     "S105",    # good idea but too many false positives on non-passwords
     "S106",    # good idea but too many false positives on non-passwords
-    "SLOT000", # Pydantic str enums can't use slots, but wanted for rendering model
+    "S107",    # good idea but too many false positives on non-passwords
+    "S603",    # not going to manually mark every subprocess call as reviewed
+    "S607",    # using PATH is not a security vulnerability
     "SIM102",  # sometimes the formatting of nested if statements is clearer
     "SIM117",  # sometimes nested with contexts are clearer
-    "T201",    # allow print
     "TCH001",  # we decided to not maintain separate TYPE_CHECKING blocks
     "TCH002",  # we decided to not maintain separate TYPE_CHECKING blocks
     "TCH003",  # we decided to not maintain separate TYPE_CHECKING blocks
-    "TD002",   # skip checking TODOs
-    "TD003",   # skip checking TODOs
-    "TD004",   # skip requiring TODOs to have issue links
     "TID252",  # if we're going to use relative imports, use them always
     "TRY003",  # good general advice but lint is way too aggressive
+    "TRY301",  # sometimes raising exceptions inside try is the best flow
+
+    # The following settings should be disabled when using ruff format
+    # per https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules
+    "W191",
+    "E111",
+    "E114",
+    "E117",
+    "D206",
+    "D300",
+    "Q000",
+    "Q001",
+    "Q002",
+    "Q003",
+    "COM812",
+    "COM819",
+    "ISC001",
+    "ISC002",
 ]
 select = ["ALL"]
-target-version = "py311"
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
+"src/timessquare/config.py" = [
+    "FBT001",  # Pydantic validators take boolean arguments
+]
 "src/timessquare/handlers/**" = [
     "D103",    # FastAPI handlers should not have docstrings
 ]
 "tests/**" = [
+    "C901",    # tests are allowed to be complex, sometimes that's convenient
+    "D101",    # tests don't need docstrings
     "D103",    # tests don't need docstrings
     "PLR0915", # tests are allowed to be long, sometimes that's convenient
     "PT012",   # way too aggressive about limiting pytest.raises blocks
     "S101",    # tests should use assert
+    "S106",    # tests are allowed to hard-code dummy passwords
     "SLF001",  # tests are allowed to access private members
-]
-"src/timessquare/config.py" = [
-    "FBT001",  # positional booleans needed for Pydantic validators
-]
-"src/timessquare/dbschema/**" = [
-    "UP006",   # use typing for JSON columns
-    "UP035",   # use typing for JSON columns
-]
-"src/timessquare/domain/github*.py" = [
-    "N815",    # allow mixed case variables on GitHub models
-]
-"src/timessquare/domain/githubcheckout.py" = [
-    "C901",    # permit complexity right now in find_notebooks method
-]
-"src/timessquare/domain/githubcheckrun.py" = [
-    "PTH122"   # allow os.path in GitHub check run for now
-]
-"src/timessquare/domain/page.py" = [
-    "C901",     # permit complexity
-    "PLR0911", # allow long methods
-    "PLR0912", # allow long methods
-]
-"src/timessquare/services/githubrepo.py" = [
-    "C901",     # permit complexity
-    "PLR0911",  # allow long methods
-    "PLR0912",  # allow long methods
-    "PLR0915",  # allow long methods
+    "T201",    # tests are allowed to use print
 ]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 known-first-party = ["timessquare", "tests"]
 split-on-trailing-comma = false
 
-[tool.ruff.flake8-bugbear]
-extend-immutable-calls = [
-    "fastapi.Form",
-    "fastapi.Header",
-    "fastapi.Depends",
-    "fastapi.Path",
-    "fastapi.Query",
-]
-
 # These are too useful as attributes or methods to allow the conflict with the
 # built-in to rule out their use.
-[tool.ruff.flake8-builtins]
+[tool.ruff.lint.flake8-builtins]
 builtins-ignorelist = [
     "all",
     "any",
@@ -218,29 +202,9 @@ builtins-ignorelist = [
     "type",
 ]
 
-[tool.ruff.flake8-pytest-style]
+[tool.ruff.lint.flake8-pytest-style]
 fixture-parentheses = false
 mark-parentheses = false
 
-[tool.ruff.pep8-naming]
-classmethod-decorators = [
-    "pydantic.root_validator",
-    "pydantic.validator",
-]
-
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "numpy"
-
-
-[tool.scriv]
-categories = [
-    "Backwards-incompatible changes",
-    "New features",
-    "Bug fixes",
-    "Other changes",
-]
-entry_title_template = "{{ version }} ({{ date.strftime('%Y-%m-%d') }})"
-format = "md"
-md_header_level = "2"
-new_fragment_template = "file:changelog.d/_template.md.jinja"
-skip_fragments = "_template.md.jinja"