Create venv rules for mypy binary

**Why**

The current method of symlinking in type stubs works on the few tests we have here, but suffers from extremely verbose outputs, including third party packages or generated code that you have no control over.

Instead, in order to mimic mypy setups that don't use bazel and achieve parity on stub discovery, we create a virtual env and use submodule calls to invoke mypy. As a result we can clean up all our custom starlark for stub discovery, as well as mypy executable rules, and tighten up our mypy config.

commit-id:871b27cd

Author: alexmirrington

.gitignore

@@ -102,7 +102,6 @@ celerybeat.pid
 .env
 .venv
 env/
-venv/
 ENV/
 env.bak/
 venv.bak/

.mypy.ini

@@ -4,13 +4,3 @@ explicit_package_bases = true
 namespace_packages = true
 show_error_codes = true
 strict = true
-
-# TODO(alexmirrington): Automatically ignore errors from symlinked stub packages
-[mypy-grpc.*]
-ignore_errors = True
-
-[mypy-google.*]
-ignore_errors = True
-
-[mypy-pandas.*]
-ignore_errors = True

.tool-versions

@@ -0,0 +1 @@
+python 3.11.7

BUILD.bazel

@@ -1,9 +1,10 @@
 load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@rules_mypy_pip_deps//:requirements.bzl", "all_whl_requirements", "requirement")
+load("@rules_mypy_pip_deps//:requirements.bzl", "all_requirements", "all_whl_requirements", "requirement")
 load("@rules_python//python:pip.bzl", "compile_pip_requirements")
 load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
 load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
 load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
+load("@rules_pyvenv//:venv.bzl", "py_venv")
 
 exports_files([
     ".mypy.ini",
@@ -39,7 +40,6 @@ gazelle_python_manifest(
     modules_mapping = ":modules_map",
     pip_repository_name = "rules_mypy_pip_deps",
     requirements = "//:requirements_lock.txt",
-    use_pip_repository_aliases = False,
 )
 
 # gazelle:resolve py tests.proto.example.v1.example_pb2 //tests/proto/example/v1
@@ -48,3 +48,9 @@ gazelle(
     data = GAZELLE_PYTHON_RUNTIME_DEPS,
     gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
 )
+
+py_venv(
+    name = "venv",
+    venv_location = ".venv",
+    deps = all_requirements,
+)

Makefile

@@ -11,6 +11,10 @@ repin:
 	bazel run //:requirements.update
 	bazel run //:gazelle_python_manifest.update
 
+.PHONY: venv
+venv:
+	bazel run //:venv
+
 .PHONY: generate
 generate:
 	bazel run //:gazelle

WORKSPACE

@@ -77,7 +77,6 @@ rules_mypy_py_deps(
     requirements_lock = "//:requirements_lock.txt",
 )
 
-# TODO(alexmirrington): Move into above macro
 load("@rules_mypy_pip_deps//:requirements.bzl", install_pip_deps = "install_deps")
 
 install_pip_deps()

pip.bzl

@@ -20,5 +20,3 @@ def rules_mypy_py_deps(requirements_lock, python_interpreter_target):
             python_interpreter_target = python_interpreter_target,
             requirements_lock = requirements_lock,
         )
-
-        # install_pip_deps()

repositories.bzl

@@ -8,9 +8,9 @@ def rules_mypy_repos():
     maybe(
         http_archive,
         name = "rules_python",
-        sha256 = "c68bdc4fbec25de5b5493b8819cfc877c4ea299c0dcb15c244c5a00208cde311",
-        strip_prefix = "rules_python-0.31.0",
-        url = "https://github.com/bazelbuild/rules_python/releases/download/0.31.0/rules_python-0.31.0.tar.gz",
+        sha256 = "3b8b4cdc991bc9def8833d118e4c850f1b7498b3d65d5698eea92c3528b8cf2c",
+        strip_prefix = "rules_python-0.30.0",
+        url = "https://github.com/bazelbuild/rules_python/releases/download/0.30.0/rules_python-0.30.0.tar.gz",
     )
 
     maybe(
@@ -23,6 +23,14 @@ def rules_mypy_repos():
         ],
     )
 
+    maybe(
+        http_archive,
+        name = "rules_pyvenv",
+        sha256 = "3a3cc6e211850178de02b618d301f3f39d1a9cddb54d499d816ff9ea835a2834",
+        strip_prefix = "rules_pyvenv-1.2",
+        url = "https://github.com/cedarai/rules_pyvenv/archive/refs/tags/v1.2.tar.gz",
+    )
+
 def rules_mypy_internal_repos():
     """Install all bazel dependencies required to build and develop the project."""
     http_archive(
@@ -45,9 +53,9 @@ def rules_mypy_internal_repos():
 
     http_archive(
         name = "rules_python_gazelle_plugin",
-        sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036",
-        strip_prefix = "rules_python-0.25.0/gazelle",
-        url = "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz",
+        sha256 = "3b8b4cdc991bc9def8833d118e4c850f1b7498b3d65d5698eea92c3528b8cf2c",
+        strip_prefix = "rules_python-0.30.0/gazelle",
+        url = "https://github.com/bazelbuild/rules_python/releases/download/0.30.0/rules_python-0.30.0.tar.gz",
     )
 
     http_archive(
@@ -56,3 +64,10 @@ def rules_mypy_internal_repos():
         strip_prefix = "rules_proto_grpc-4.6.0",
         urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.6.0/rules_proto_grpc-4.6.0.tar.gz"],
     )
+
+    http_archive(
+        name = "rules_pyvenv",
+        sha256 = "3a3cc6e211850178de02b618d301f3f39d1a9cddb54d499d816ff9ea835a2834",
+        strip_prefix = "rules_pyvenv-1.2",
+        url = "https://github.com/cedarai/rules_pyvenv/archive/refs/tags/v1.2.tar.gz",
+    )

rules/mypy/BUILD.bazel

@@ -1,23 +1,12 @@
 load("@rules_mypy_pip_deps//:requirements.bzl", "all_requirements")
-load("@rules_python//python:defs.bzl", "py_binary")
-load("//rules/mypy:stubs.bzl", "py_stubs")
+load("//rules/venv:venv.bzl", "py_venv")
 
 exports_files([
     "mypy.bash.tpl",
-    "__main__.py",
 ])
 
-py_binary(
-    name = "__main__",
-    srcs = ["__main__.py"],
-    main = "__main__.py",
-    visibility = ["//visibility:public"],
-    deps = ["//config:mypy"],  # keep
-)
-
-# Make PEP-561 stubs-only packages available for type-checking.
-py_stubs(
-    name = "stubs",
+py_venv(
+    name = "venv",
     visibility = ["//visibility:public"],
     deps = all_requirements,  # keep
 )

rules/mypy/__main__.py

@@ -9,25 +9,13 @@ main() {
   local output
   local report_file
   local status
-  local root
-  local mypy
 
   report_file="{OUTPUT}"
-  root="{MYPY_ROOT}/"
-  mypy="{MYPY_EXE}"
-
-  # TODO(Jonathon): Consider UX improvements using https://mypy.readthedocs.io/en/stable/command_line.html#configuring-error-messages
 
   export MYPYPATH="$(pwd):{MYPYPATH_PATH}"
 
-  # Workspace rules run in a different location from aspect rules. Here we
-  # normalize if the external source isn't found.
-  if [ ! -f $mypy ]; then
-    mypy=${mypy#${root}}
-  fi
-
   set +o errexit
-  output=$($mypy {VERBOSE_OPT} --bazel {PACKAGE_ROOTS} --config-file {MYPY_INI_PATH} --cache-map {CACHE_MAP_TRIPLES} -- {SRCS} 2>&1)
+  output=$({PY_INTERPRETER} -m mypy {VERBOSE_OPT}  --bazel {PACKAGE_ROOTS} --config-file {MYPY_INI_PATH} --cache-map {CACHE_MAP_TRIPLES} -- {SRCS} 2>&1)
   status=$?
   set -o errexit
 
@@ -36,7 +24,9 @@ main() {
   fi
 
   if [[ $status -ne 0 ]]; then
-    echo "${output}" # Show MyPy's error to end-user via Bazel's console logging
+    # Show type checking errors to end-user via Bazel's console logging
+    # TODO(alexmirrington): Consider UX improvements using https://mypy.readthedocs.io/en/stable/command_line.html#configuring-error-messages
+    echo "${output}"
     exit 1
   fi
 

rules/mypy/mypy.bash.tpl

@@ -5,7 +5,7 @@ Initial source taken from here: https://github.com/bazel-contrib/bazel-mypy-inte
 
 load("@bazel_skylib//lib:sets.bzl", "sets")
 load("@bazel_skylib//lib:shell.bzl", "shell")
-load("//rules/mypy:stubs.bzl", "PyStubsInfo")
+load("//rules/venv:venv.bzl", "PyVenvInfo")
 
 MyPyAspectInfo = provider(
     "TODO: documentation",
@@ -23,23 +23,18 @@ MYPY_DEBUG = False
 VALID_EXTENSIONS = ["py", "pyi"]
 
 DEFAULT_ATTRS = {
-    "_mypy_cli": attr.label(
-        default = Label("//rules/mypy:__main__"),
-        executable = True,
-        cfg = "exec",
-    ),
     "_mypy_config": attr.label(
         default = Label("//config:mypy_config"),
         allow_single_file = True,
     ),
-    "_stubs": attr.label(
-        default = Label("//rules/mypy:stubs"),
-        executable = False,
-    ),
     "_template": attr.label(
         default = Label("//rules/mypy:mypy.bash.tpl"),
         allow_single_file = True,
     ),
+    "_venv": attr.label(
+        default = Label("//rules/mypy:venv"),
+        executable = False,
+    ),
 }
 
 def _sources_to_cache_map_triples(srcs, is_aspect):
@@ -133,8 +128,8 @@ def _mypy_rule_impl(ctx, is_aspect = False):
         base_rule = ctx.rule
 
     mypy_config_file = ctx.file._mypy_config
-    mypypath_parts = [ctx.attr._stubs[PyStubsInfo].stubs_directory]
 
+    mypypath_parts = []
     direct_src_files = []
 
     transitive_srcs_depsets = []
@@ -178,10 +173,8 @@ def _mypy_rule_impl(ctx, is_aspect = False):
     # the project version of mypy however, other rules should fall back on their
     # relative runfiles.
     runfiles = ctx.runfiles(
-        files = src_files + ctx.attr._stubs[PyInfo].transitive_sources.to_list() + [mypy_config_file],
+        files = src_files + [ctx.attr._venv[PyVenvInfo].venv_dir] + [mypy_config_file],
     )
-    if not is_aspect:
-        runfiles = runfiles.merge(ctx.attr._mypy_cli.default_runfiles)
 
     src_root_paths = sets.to_list(
         sets.make([f.root.path for f in src_files]),
@@ -193,14 +186,13 @@ def _mypy_rule_impl(ctx, is_aspect = False):
         substitutions = {
             "{CACHE_MAP_TRIPLES}": " ".join(_sources_to_cache_map_triples(src_files, is_aspect)),
             "{MYPYPATH_PATH}": mypypath if mypypath else "",
-            "{MYPY_EXE}": ctx.executable._mypy_cli.path,
             "{MYPY_INI_PATH}": mypy_config_file.path,
-            "{MYPY_ROOT}": ctx.executable._mypy_cli.root.path,
             "{OUTPUT}": out.path if out else "",
             "{PACKAGE_ROOTS}": " ".join([
                 "--package-root " + shell.quote(path or ".")
                 for path in src_root_paths
             ]),
+            "{PY_INTERPRETER}": "%s/bin/python3" % ctx.attr._venv[PyVenvInfo].venv_dir.path,
             "{SRCS}": " ".join([
                 shell.quote(f.path) if is_aspect else shell.quote(f.short_path)
                 for f in src_files
@@ -236,7 +228,7 @@ def _mypy_aspect_impl(_, ctx):
     ctx.actions.run(
         outputs = [aspect_info.out],
         inputs = info.default_runfiles.files,
-        tools = [ctx.executable._mypy_cli],
+        tools = [],
         executable = aspect_info.exe,
         mnemonic = "MyPy",
         progress_message = "Type-checking %s" % ctx.label,

rules/mypy/mypy.bzl

@@ -0,0 +1,3 @@
+exports_files([
+    "venv.bash.tpl",
+])