Add hidden support for Python 3.13. (#2318)

This uses a patched version of Pip as well as a new build system to
affect spinning up CI for the Python 3.13 release in October ahead of
time without leaking details of this testing-only setup to the
production distribution.

.github/workflows/ci.yml

@@ -57,6 +57,7 @@ jobs:
           - py311-pip22_3_1
           - py311-pip23_1_2
           - py312-pip23_3_2
+          - py313-pip24_0
           - pypy310-pip20
           - pypy310-pip22_3_1
           - pypy310-pip23_1_2
@@ -67,6 +68,7 @@ jobs:
           - py311-pip22_3_1-integration
           - py311-pip23_1_2-integration
           - py312-pip23_3_2-integration
+          - py313-pip24_0-integration
           - pypy310-pip20-integration
           - pypy310-pip22_3_1-integration
           - pypy310-pip23_1_2-integration

.gitignore

@@ -1,17 +1,9 @@
-*~
 *.pyc
 *.pyo
-*.egg-info
-/.cache
-/venv
-/build/*
-/dist/*
+*~
+
+/.mypy_cache/
+/.tox/
+/dist/
 /docs/_build
-/.tox
-/.idea
-/.coverage*
-/htmlcov
-/.pyenv_test
-.cache/*
-.eggs/*
-/.pytest_cache
+

MANIFEST.in

@@ -0,0 +1,7 @@
+include CHANGES.md dtox.sh mypy.ini tox.ini
+graft docker
+graft docs
+graft scripts
+graft testing
+graft tests
+global-exclude *.pyc *.pyo

docker/base/install_pythons.sh

@@ -16,13 +16,14 @@ PYENV_VERSIONS=(
   3.9.18
   3.10.13
   3.12.1
-  pypy2.7-7.3.13
+  3.13.0a2
+  pypy2.7-7.3.14
   pypy3.5-7.0.0
   pypy3.6-7.3.3
   pypy3.7-7.3.9
   pypy3.8-7.3.11
-  pypy3.9-7.3.13
-  pypy3.10-7.3.13
+  pypy3.9-7.3.14
+  pypy3.10-7.3.14
 )
 
 git clone https://github.com/pyenv/pyenv.git "${PYENV_ROOT}" && (
@@ -31,13 +32,13 @@ git clone https://github.com/pyenv/pyenv.git "${PYENV_ROOT}" && (
 PATH="${PATH}:${PYENV_ROOT}/bin"
 
 for version in "${PYENV_VERSIONS[@]}"; do
-  if [[ "${version}" == "pypy2.7-7.3.13" ]]; then
-    # Installation of pypy2.7-7.3.13 fails like so without adjusting the version of get-pip it
+  if [[ "${version}" == "pypy2.7-7.3.14" ]]; then
+    # Installation of pypy2.7-7.3.14 fails like so without adjusting the version of get-pip it
     # uses:
-    #  $ pyenv install pypy2.7-7.3.13
-    #  Downloading pypy2.7-v7.3.13-linux64.tar.bz2...
-    #  -> https://downloads.python.org/pypy/pypy2.7-v7.3.13-linux64.tar.bz2
-    #  Installing pypy2.7-v7.3.13-linux64...
+    #  $ pyenv install pypy2.7-7.3.14
+    #  Downloading pypy2.7-v7.3.14-linux64.tar.bz2...
+    #  -> https://downloads.python.org/pypy/pypy2.7-v7.3.14-linux64.tar.bz2
+    #  Installing pypy2.7-v7.3.14-linux64...
     #  Installing pip from https://bootstrap.pypa.io/get-pip.py...
     #  error: failed to install pip via get-pip.py
     #  ...

pex/__init__.py

@@ -1,10 +1,2 @@
 # Copyright 2014 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-"""The PEX packaging toolchain."""  # N.B.: Flit uses this as our distribution description.
-
-from __future__ import absolute_import
-
-from .version import __version__ as __pex_version__
-
-__version__ = __pex_version__  # N.B.: Flit uses this as out distribution version.

pex/bin/pex.py

@@ -40,7 +40,7 @@
 from pex.resolve.configured_resolve import resolve
 from pex.resolve.requirement_configuration import RequirementConfiguration
 from pex.resolve.resolvers import Unsatisfiable
-from pex.result import ResultError, catch, try_
+from pex.result import Error, ResultError, catch, try_
 from pex.targets import Targets
 from pex.tracer import TRACER
 from pex.typing import TYPE_CHECKING, cast
@@ -958,15 +958,21 @@ def _compatible_with_current_platform(interpreter, platforms):
 def main(args=None):
     args = args[:] if args else sys.argv[1:]
     args = [transform_legacy_arg(arg) for arg in args]
-    parser = configure_clp()
+
+    parser = catch(configure_clp)
+    if isinstance(parser, Error):
+        die(str(parser))
 
     try:
         separator = args.index("--")
         args, cmdline = args[:separator], args[separator + 1 :]
     except ValueError:
         args, cmdline = args, []
 
-    options = parser.parse_args(args=args)
+    options = catch(parser.parse_args, args=args)
+    if isinstance(options, Error):
+        die(str(options))
+
     try:
         with global_environment(options) as env:
             requirement_configuration = requirement_options.configure(options)

pex/build_system/pep_517.py

@@ -6,7 +6,6 @@
 import json
 import os
 import subprocess
-import sys
 from textwrap import dedent
 
 from pex import third_party
@@ -118,8 +117,6 @@ def _invoke_build_hook(
     hook_args=(),  # type: Iterable[Any]
     hook_extra_requirements=None,  # type: Optional[Iterable[str]]
     hook_kwargs=None,  # type: Optional[Mapping[str, Any]]
-    stdout=None,  # type: Optional[int]
-    stderr=None,  # type: Optional[int]
     pip_version=None,  # type: Optional[PipVersionValue]
 ):
     # type: (...) -> Union[SpawnedJob[Any], Error]
@@ -183,8 +180,8 @@ def _invoke_build_hook(
             args=args,
             env=build_system.env,
             cwd=project_directory,
-            stdout=stdout if stdout is not None else sys.stderr.fileno(),
-            stderr=stderr if stderr is not None else sys.stderr.fileno(),
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
         )
         return SpawnedJob.file(
             Job(command=args, process=process),

pex/interpreter.py

@@ -217,7 +217,11 @@ def get(cls, binary=None):
             python_tag=preferred_tag.interpreter,
             abi_tag=preferred_tag.abi,
             platform_tag=preferred_tag.platform,
-            version=sys.version_info[:3],
+            version=cast("Tuple[int, int, int]", tuple(sys.version_info[:3])),
+            pypy_version=cast(
+                "Optional[Tuple[int, int, int]]",
+                tuple(getattr(sys, "pypy_version_info", ())[:3]) or None,
+            ),
             supported_tags=supported_tags,
             env_markers=MarkerEnvironment.default(),
             configured_macosx_deployment_target=configured_macosx_deployment_target,
@@ -227,9 +231,12 @@ def get(cls, binary=None):
     def decode(cls, encoded):
         TRACER.log("creating PythonIdentity from encoded: %s" % encoded, V=9)
         values = json.loads(encoded)
-        if len(values) != 15:
+        if len(values) != 16:
             raise cls.InvalidError("Invalid interpreter identity: %s" % encoded)
 
+        version = tuple(values.pop("version"))
+        pypy_version = tuple(values.pop("pypy_version") or ()) or None
+
         supported_tags = values.pop("supported_tags")
 
         def iter_tags():
@@ -244,6 +251,8 @@ def iter_tags():
 
         env_markers = MarkerEnvironment(**values.pop("env_markers"))
         return cls(
+            version=cast("Tuple[int, int, int]", version),
+            pypy_version=pypy_version,
             supported_tags=iter_tags(),
             configured_macosx_deployment_target=configured_macosx_deployment_target,
             env_markers=env_markers,
@@ -270,7 +279,8 @@ def __init__(
         python_tag,  # type: str
         abi_tag,  # type: str
         platform_tag,  # type: str
-        version,  # type: Iterable[int]
+        version,  # type: Tuple[int, int, int]
+        pypy_version,  # type: Optional[Tuple[int, int, int]]
         supported_tags,  # type: Iterable[tags.Tag]
         env_markers,  # type: MarkerEnvironment
         configured_macosx_deployment_target,  # type: Optional[str]
@@ -291,7 +301,8 @@ def __init__(
         self._python_tag = python_tag
         self._abi_tag = abi_tag
         self._platform_tag = platform_tag
-        self._version = tuple(version)
+        self._version = version
+        self._pypy_version = pypy_version
         self._supported_tags = CompatibilityTags(tags=supported_tags)
         self._env_markers = env_markers
         self._configured_macosx_deployment_target = configured_macosx_deployment_target
@@ -310,6 +321,7 @@ def encode(self):
             abi_tag=self._abi_tag,
             platform_tag=self._platform_tag,
             version=self._version,
+            pypy_version=self._pypy_version,
             supported_tags=[
                 (tag.interpreter, tag.abi, tag.platform) for tag in self._supported_tags
             ],
@@ -376,7 +388,22 @@ def version(self):
 
         Consistent with `sys.version_info`, the tuple corresponds to `<major>.<minor>.<micro>`.
         """
-        return cast("Tuple[int, int, int]", self._version)
+        return self._version
+
+    @property
+    def pypy_version(self):
+        # type: () -> Optional[Tuple[int, int, int]]
+        """The PyPy implementation version as a normalized tuple.
+
+        Only present for PyPy interpreters and, consistent with `sys.pypy_version_info`, the tuple
+        corresponds to `<major>.<minor>.<micro>`.
+        """
+        return self._pypy_version
+
+    @property
+    def is_pypy(self):
+        # type: () -> bool
+        return bool(self._pypy_version)
 
     @property
     def version_str(self):
@@ -426,11 +453,7 @@ def binary_name(self, version_components=2):
     def hashbang(self):
         # type: () -> str
         return "#!/usr/bin/env {}".format(
-            self.binary_name(
-                version_components=0
-                if self._interpreter_name == "PyPy" and self.version[0] == 2
-                else 2
-            )
+            self.binary_name(version_components=0 if self.is_pypy and self.version[0] == 2 else 2)
         )
 
     @property
@@ -1246,7 +1269,7 @@ def resolve_base_interpreter(self):
         version = self._identity.version
         abi_tag = self._identity.abi_tag
 
-        prefix = "pypy" if self._identity.interpreter == "PyPy" else "python"
+        prefix = "pypy" if self.is_pypy else "python"
         suffixes = ("{}.{}".format(version[0], version[1]), str(version[0]), "")
         candidate_binaries = tuple("{}{}".format(prefix, suffix) for suffix in suffixes)
 
@@ -1298,6 +1321,11 @@ def identity(self):
         # type: () -> PythonIdentity
         return self._identity
 
+    @property
+    def is_pypy(self):
+        # type: () -> bool
+        return self._identity.is_pypy
+
     @property
     def python(self):
         return self._identity.python

pex/pip/version.py

@@ -5,6 +5,7 @@
 
 import os
 import sys
+from textwrap import dedent
 
 from pex.dist_metadata import Requirement
 from pex.enum import Enum
@@ -114,14 +115,29 @@ def __get__(self, obj, objtype=None):
                     self._default = preferred_version
                     break
             if self._default is None:
-                self._default = max(
-                    (
-                        version
-                        for version in PipVersionValue._iter_values()
-                        if not version.hidden and version.requires_python_applies(current_version)
-                    ),
-                    key=lambda pv: pv.version,
+                applicable_versions = tuple(
+                    version
+                    for version in PipVersionValue._iter_values()
+                    if not version.hidden and version.requires_python_applies(current_version)
                 )
+                if not applicable_versions:
+                    raise ValueError(
+                        dedent(
+                            """\
+                            No version of Pip supported by Pex works with {python}.
+                            The supported Pip versions are:
+                            {versions}
+                            """
+                        ).format(
+                            python=sys.executable,
+                            versions=", ".join(
+                                version.value
+                                for version in PipVersionValue._iter_values()
+                                if not version.hidden
+                            ),
+                        )
+                    )
+                self._default = max(applicable_versions, key=lambda pv: pv.version)
         return self._default
 
 
@@ -211,7 +227,7 @@ def values(cls):
         version="23.2",
         setuptools_version="68.0.0",
         wheel_version="0.40.0",
-        requires_python=">=3.7",
+        requires_python=">=3.7,<3.13",
     )
 
     v23_3_1 = PipVersionValue(
@@ -220,7 +236,7 @@ def values(cls):
         # date) but 68.0.0 is the last setuptools version to support 3.7.
         setuptools_version="68.0.0",
         wheel_version="0.41.2",
-        requires_python=">=3.7",
+        requires_python=">=3.7,<3.13",
     )
 
     v23_3_2 = PipVersionValue(
@@ -229,7 +245,22 @@ def values(cls):
         # date) but 68.0.0 is the last setuptools version to support 3.7.
         setuptools_version="68.0.0",
         wheel_version="0.42.0",
+        requires_python=">=3.7,<3.13",
+    )
+
+    # This is https://github.com/pypa/pip/pull/12462 which is approved but not yet merged or
+    # released. It allows testing Python 3.13 pre-releases but should not be used by the public; so
+    # we keep it hidden.
+    v24_0_dev0_patched = PipVersionValue(
+        name="24.0.dev0-patched",
+        version="24.0.dev0+patched",
+        requirement=(
+            "pip @ git+https://github.com/jsirois/pip@0257c9422f7bb99a6f319b54f808a5c50339be6c"
+        ),
+        setuptools_version="69.0.3",
+        wheel_version="0.42.0",
         requires_python=">=3.7",
+        hidden=True,
     )
 
     VENDORED = v20_3_4_patched

pex/vendor/__init__.py

@@ -149,9 +149,16 @@ def create_packages(self):
         :class:`pex.third_party.VendorImporter`.
         """
         if not self.rewrite:
-            # The extra package structure is only required for vendored code used via import
+            # The extra package structure is only required by Pex for vendored code used via import
             # rewrites.
-            return
+
+            # N.B.: Although we've historically early-returned here, the switch from flit to
+            # setuptools for our build backend necessitates all vendored dists are seen as part of
+            # the `pex` package tree by setuptools to get all vendored code properly included in
+            # our distribution.
+            # TODO(John Sirois): re-introduce early return once it is no longer foils our build
+            #  process.
+            pass
 
         for index, _ in enumerate(self._subpath_components):
             relpath = _PACKAGE_COMPONENTS + self._subpath_components[: index + 1] + ["__init__.py"]

pex/vendor/_vendored/pip/.layout.json

@@ -1 +1 @@
-{"fingerprint": "120267325b80f5c4b4adac019eb6617ab3319395c043d2871eedf70dd6ae2954", "record_relpath": "pip-20.3.4.dist-info/RECORD", "stash_dir": ".prefix"}
+{"fingerprint": "69c1ee1f3c238f9f7a88645a63f68c3c5802b6e3d349612fde5fe2b8c692a668", "record_relpath": "pip-20.3.4.dist-info/RECORD", "stash_dir": ".prefix"}

pex/venv/installer.py

@@ -678,6 +678,7 @@ def sys_executable_paths():
                     # These are _not_ used at runtime, but are present under testing / CI and
                     # simplest to add an exception for here and not warn about in CI runs.
                     "_PEX_PIP_VERSION",
+                    "_PEX_REQUIRES_PYTHON",
                     "_PEX_TEST_DEV_ROOT",
                     "_PEX_TEST_PROJECT_DIR",
                     "_PEX_USE_PIP_CONFIG",