Fix python linting and formatting (#166)

* Added new workflow for enabling Python linting in CI
* Adjust config so that linting and format works from root
  * Moved mypy config into root so it's picked up globally.
  * Moved .flake8 config to root
* Make mypy green
  * Fix typos in dicom example
  * Fixed mypy linting issues in rerun_sdk and color_conversion
  * Added numpy to lint requirements for typing plugin
Co-authored-by: Emil Ernerfeldt <emil@rerun.io>

Author: John Hughes

rerun_py/.flake8 .flake8

@@ -1,5 +1,5 @@
 [flake8]
-max-line-length = 88
+max-line-length = 120
 ban-relative-imports = true
 docstring-convention = all
 extend-ignore =
@@ -17,3 +17,6 @@ extend-exclude =
     # Automatically generated test artifacts
     venv/,
     target/,
+
+    # Disabled for now TODO(john)
+    examples/,

.github/workflows/lint-python.yml

@@ -0,0 +1,44 @@
+name: Lint Python
+
+on:
+  pull_request:
+    paths:
+      - "rerun_py/**"
+      - "examples/**"
+      - ".github/workflows/lint-python.yml"
+  push:
+    branches:
+      - "main"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  main:
+    name: Lint Python
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: extractions/setup-just@v1
+        with:
+          just-version: 1.5
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+          cache: 'pip'
+          cache-dependency-path: 'rerun_py/requirements-lint.txt'
+
+      - name: Install Python dependencies
+        run: |
+          pip install --upgrade pip
+          pip install -r rerun_py/requirements-lint.txt
+
+      - name: Lint Python
+        run: |
+          just py-lint

.isort.cfg

@@ -0,0 +1,2 @@
+[settings]
+profile=black

.mypy.ini

@@ -0,0 +1,7 @@
+[mypy]
+files = rerun_py/rerun_sdk, rerun_py/tests
+namespace_packages = True
+show_error_codes = True
+strict = True
+enable_error_code = redundant-expr, truthy-bool, ignore-without-code
+plugins = numpy.typing.mypy_plugin

.vscode/settings.json

@@ -12,5 +12,7 @@
         "target/**": true
     },
     "files.autoGuessEncoding": true,
-    "python.formatting.provider": "black"
+    "python.formatting.provider": "black",
+    "python.linting.mypyEnabled": true,
+    "python.linting.enabled": true
 }

examples/car/main.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python3
 """Shows how to use the Rerun SDK."""
 
-from dataclasses import dataclass
+import argparse
 import os
+from dataclasses import dataclass
 from pathlib import Path
 from time import sleep
 from typing import Final, Iterator, Tuple
 
-import argparse
 import cv2  # type: ignore
 import numpy as np
-
 import rerun_sdk as rerun
 
 CAMERA_GLB: Final = Path(os.path.dirname(__file__)).joinpath(

examples/deep_sdf/download_dataset.py

@@ -2,9 +2,10 @@
 
 import io
 import os
-import requests
-from pathlib import Path
 import zipfile
+from pathlib import Path
+
+import requests
 
 
 def download_mcguire_sample(package, name):

examples/deep_sdf/main.py

@@ -34,19 +34,17 @@
 
 import argparse
 import os
+from pathlib import Path
+from timeit import default_timer as timer
+from typing import Tuple, cast
 
 import mesh_to_sdf
 import numpy as np
 import rerun_sdk as rerun
 import trimesh
-
-from pathlib import Path
-from timeit import default_timer as timer
-from typing import Tuple, cast
-
+from rerun_sdk import LogLevel, MeshFormat
 from scipy.spatial.transform import Rotation as R
 from trimesh import Trimesh
-from rerun_sdk import MeshFormat, LogLevel
 
 
 def announcement(body: str):

examples/dicom/download_dataset.py

@@ -2,13 +2,12 @@
 
 import io
 import os
+import zipfile
 from pathlib import Path
 from typing import Final
-import zipfile
 
 import requests
 
-
 DATASET_DIR: Final = Path(os.path.dirname(__file__)) / "dataset"
 
 

examples/dicom/main.py

@@ -1,26 +1,26 @@
 #!/usr/bin/env python3
-"""Example using a vicom MRI scan.
+"""Example using a dicom MRI scan.
 
 Setup:
 ``` sh
-python3 examples/vicom/download_dataset.py
+python3 examples/dicom/download_dataset.py
 ```
 
 Run:
 ``` sh
-python3 examples/vicom/main.py
+python3 examples/dicom/main.py
 ```
 """
 
 import argparse
-from typing import Final, Iterable, Tuple
-import pydicom as dicom
-import dicom_numpy   # type: ignore
-import numpy as np
 import os
 from pathlib import Path
-import numpy.typing as npt
+from typing import Final, Iterable, Tuple
 
+import dicom_numpy  # type: ignore
+import numpy as np
+import numpy.typing as npt
+import pydicom as dicom
 import rerun_sdk as rerun
 
 DATASET_DIR: Final = Path(os.path.dirname(__file__)) / "dataset"
@@ -42,7 +42,7 @@ def list_dicom_files(dir: Path) -> Iterable[Path]:
                 yield Path(path) / f
 
 
-def read_and_log_vicom_dataset():
+def read_and_log_dicom_dataset():
     dicom_files = list_dicom_files(DATASET_DIR)
     voxels_volume, _ = extract_voxel_data(dicom_files)
     voxels_volume = voxels_volume.astype('uint16') # the data is i16, but in range [0, 536].
@@ -69,7 +69,7 @@ def main() -> None:
         # which is `127.0.0.1:9876`.
         rerun.connect(args.addr)
 
-    read_and_log_vicom_dataset()
+    read_and_log_dicom_dataset()
 
     if args.save is not None:
         rerun.save(args.save)

examples/multiprocessing/main.py

@@ -6,6 +6,7 @@
 
 import rerun_sdk as rerun
 
+
 def task(title):
     # All processes spawned with `multiprocessing` will automatically
     # be assigned the same default recording_id.

examples/nyud/main.py

@@ -15,14 +15,14 @@
 """
 
 import argparse
+import zipfile
 from dataclasses import dataclass
+from datetime import datetime
 from pathlib import Path
 from typing import Final, Tuple
-import zipfile
-from datetime import datetime
+
 import cv2
 import numpy as np
-
 import rerun_sdk as rerun
 
 # Logging depth images is slow, so we don't log every frame

examples/objectron/download_dataset.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python3
 
-import requests
 import os
 
+import requests
+
 
 def download(url, path):
     if not os.path.exists(path):

examples/objectron/main.py

@@ -22,19 +22,23 @@
 import math
 import os
 import sys
-
-import numpy as np
-import rerun_sdk as rerun
-
 from dataclasses import dataclass
 from pathlib import Path
-from typing import List, Final, Iterator, Iterable
+from typing import Final, Iterable, Iterator, List
 
+import numpy as np
+import rerun_sdk as rerun
+from proto.objectron.proto import (
+    ARCamera,
+    ARFrame,
+    ARPointCloud,
+    FrameAnnotation,
+    Object,
+    ObjectType,
+    Sequence,
+)
 from rerun_sdk import ImageFormat
 from scipy.spatial.transform import Rotation as R
-from proto.objectron.proto import \
-    ARFrame, ARCamera, ARPointCloud, Sequence, Object, ObjectType, FrameAnnotation
-
 
 IMAGE_RESOLUTION: Final = (1440, 1920)
 GEOMETRY_FILENAME: Final = "geometry.pbdata"

justfile

@@ -22,16 +22,17 @@ py-build:
 
 # Run autoformatting
 py-format:
-    isort .
-    black .
+    black --config rerun_py/pyproject.toml .
     blackdoc .
+    isort .
     pyupgrade --py37-plus `find rerun_sdk/ tests/ -name "*.py" -type f`
     cargo fmt --all
 
 # Run linting
 py-lint:
-    #!/usr/bin/env bash
-    cd rerun_py
+    black --check --config rerun_py/pyproject.toml .
+    blackdoc --check .
+    isort --check .
     mypy
     flake8
 

rerun_py/pyproject.toml

@@ -17,7 +17,7 @@ keywords = ["computer-vision", "logging", "rerun"]
 license = { text = "MIT OR Apache-2.0" }
 requires-python = ">=3.7"
 
-dependencies = ["numpy"]
+dependencies = ["numpy==1.23"]
 
 [project.optional-dependencies]
 tests = ["pytest==7.1.2"]
@@ -30,12 +30,7 @@ repository = "https://github.com/rerun-io/rerun"
 [tool.maturin]
 locked = true
 
-[tool.isort]
-profile = "black"
-
-[tool.mypy]
-files = ["rerun_sdk", "tests"]
-namespace_packages = true
-show_error_codes = true
-strict = true
-enable_error_code = ["redundant-expr", "truthy-bool", "ignore-without-code"]
+[tool.black]
+line-length = 120
+target-version = ['py38']
+extend-exclude = "examples"

rerun_py/requirements-lint.txt

@@ -10,3 +10,4 @@ isort==5.10.1
 mypy==0.971
 pyupgrade==2.37.3
 types-requests==2.28.10
+numpy==1.23 # For mypy plugin