Try switching to venv for isolation

src/pip/_internal/build_env.py

@@ -6,20 +6,27 @@
 import os
 import pathlib
 import sys
-import textwrap
+import venv
 import zipfile
-from collections import OrderedDict
-from sysconfig import get_paths
 from types import TracebackType
-from typing import TYPE_CHECKING, Generator, Iterable, List, Optional, Set, Tuple, Type
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Generator,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Tuple,
+    Type,
+)
 
 from pip._vendor.certifi import where
 from pip._vendor.packaging.requirements import Requirement
 from pip._vendor.packaging.version import Version
 
 from pip import __file__ as pip_location
 from pip._internal.cli.spinners import open_spinner
-from pip._internal.locations import get_platlib, get_prefixed_libs, get_purelib
 from pip._internal.metadata import get_default_environment, get_environment
 from pip._internal.utils.subprocess import call_subprocess
 from pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds
@@ -30,17 +37,6 @@
 logger = logging.getLogger(__name__)
 
 
-class _Prefix:
-    def __init__(self, path: str) -> None:
-        self.path = path
-        self.setup = False
-        self.bin_dir = get_paths(
-            "nt" if os.name == "nt" else "posix_prefix",
-            vars={"base": path, "platbase": path},
-        )["scripts"]
-        self.lib_dirs = get_prefixed_libs(path)
-
-
 @contextlib.contextmanager
 def _create_standalone_pip() -> Generator[str, None, None]:
     """Create a "standalone pip" zip file.
@@ -71,80 +67,42 @@ class BuildEnvironment:
     """Creates and manages an isolated environment to install build deps"""
 
     def __init__(self) -> None:
-        temp_dir = TempDirectory(kind=tempdir_kinds.BUILD_ENV, globally_managed=True)
-
-        self._prefixes = OrderedDict(
-            (name, _Prefix(os.path.join(temp_dir.path, name)))
-            for name in ("normal", "overlay")
+        self._temp_dir = TempDirectory(
+            kind=tempdir_kinds.BUILD_ENV, globally_managed=True
         )
-
-        self._bin_dirs: List[str] = []
-        self._lib_dirs: List[str] = []
-        for prefix in reversed(list(self._prefixes.values())):
-            self._bin_dirs.append(prefix.bin_dir)
-            self._lib_dirs.extend(prefix.lib_dirs)
-
-        # Customize site to:
-        # - ensure .pth files are honored
-        # - prevent access to system site packages
-        system_sites = {
-            os.path.normcase(site) for site in (get_purelib(), get_platlib())
-        }
-        self._site_dir = os.path.join(temp_dir.path, "site")
-        if not os.path.exists(self._site_dir):
-            os.mkdir(self._site_dir)
-        with open(
-            os.path.join(self._site_dir, "sitecustomize.py"), "w", encoding="utf-8"
-        ) as fp:
-            fp.write(
-                textwrap.dedent(
-                    """
-                import os, site, sys
-
-                # First, drop system-sites related paths.
-                original_sys_path = sys.path[:]
-                known_paths = set()
-                for path in {system_sites!r}:
-                    site.addsitedir(path, known_paths=known_paths)
-                system_paths = set(
-                    os.path.normcase(path)
-                    for path in sys.path[len(original_sys_path):]
-                )
-                original_sys_path = [
-                    path for path in original_sys_path
-                    if os.path.normcase(path) not in system_paths
-                ]
-                sys.path = original_sys_path
-
-                # Second, add lib directories.
-                # ensuring .pth file are processed.
-                for path in {lib_dirs!r}:
-                    assert not path in sys.path
-                    site.addsitedir(path)
-                """
-                ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)
+        self._venv = venv.EnvBuilder()
+        context = self._venv.ensure_directories(self._temp_dir.path)
+        self._venv.create(self._temp_dir.path)
+
+        # Copy-pasted from venv/__init__.py
+        if sys.platform == "win32":
+            libpath = os.path.join(
+                self._temp_dir.path,
+                "Lib",
+                "site-packages",
+            )
+        else:
+            libpath = os.path.join(
+                self._temp_dir.path,
+                "lib",
+                f"python{sys.version_info.major}.{sys.version_info.minor}",
+                "site-packages",
             )
 
+        self._lib_path = libpath
+        self._bin_path = context.bin_path
+        self._env_executable = context.env_exe
+        self._save_env: Dict[str, str] = {}
+
     def __enter__(self) -> None:
         self._save_env = {
-            name: os.environ.get(name, None)
-            for name in ("PATH", "PYTHONNOUSERSITE", "PYTHONPATH")
+            name: os.environ.get(name, "") for name in ("PATH", "PYTHONPATH")
         }
 
-        path = self._bin_dirs[:]
         old_path = self._save_env["PATH"]
-        if old_path:
-            path.extend(old_path.split(os.pathsep))
-
-        pythonpath = [self._site_dir]
+        new_path = os.pathsep.join(filter(None, [self._bin_path, old_path]))
 
-        os.environ.update(
-            {
-                "PATH": os.pathsep.join(path),
-                "PYTHONNOUSERSITE": "1",
-                "PYTHONPATH": os.pathsep.join(pythonpath),
-            }
-        )
+        os.environ.update({"PATH": new_path, "PYTHONPATH": self._lib_path})
 
     def __exit__(
         self,
@@ -168,11 +126,7 @@ def check_requirements(
         missing = set()
         conflicting = set()
         if reqs:
-            env = (
-                get_environment(self._lib_dirs)
-                if hasattr(self, "_lib_dirs")
-                else get_default_environment()
-            )
+            env = get_environment([self._lib_path])
             for req_str in reqs:
                 req = Requirement(req_str)
                 # We're explicitly evaluating with an empty extra value, since build
@@ -195,43 +149,36 @@ def check_requirements(
     def install_requirements(
         self,
         finder: "PackageFinder",
-        requirements: Iterable[str],
-        prefix_as_string: str,
         *,
+        requirements: Iterable[str],
         kind: str,
     ) -> None:
-        prefix = self._prefixes[prefix_as_string]
-        assert not prefix.setup
-        prefix.setup = True
+        assert self._env_executable
         if not requirements:
             return
-        with contextlib.ExitStack() as ctx:
-            pip_runnable = ctx.enter_context(_create_standalone_pip())
+        with _create_standalone_pip() as pip_runnable:
             self._install_requirements(
+                self._env_executable,
                 pip_runnable,
                 finder,
                 requirements,
-                prefix,
                 kind=kind,
             )
 
     @staticmethod
     def _install_requirements(
+        executable: str,
         pip_runnable: str,
         finder: "PackageFinder",
         requirements: Iterable[str],
-        prefix: _Prefix,
         *,
         kind: str,
     ) -> None:
         args: List[str] = [
-            sys.executable,
+            executable,
             pip_runnable,
             "install",
-            "--ignore-installed",
             "--no-user",
-            "--prefix",
-            prefix.path,
             "--no-warn-script-location",
         ]
         if logger.getEffectiveLevel() <= logging.DEBUG:
@@ -290,15 +237,11 @@ def __exit__(
     ) -> None:
         pass
 
-    def cleanup(self) -> None:
-        pass
-
     def install_requirements(
         self,
         finder: "PackageFinder",
-        requirements: Iterable[str],
-        prefix_as_string: str,
         *,
+        requirements: Iterable[str],
         kind: str,
     ) -> None:
         raise NotImplementedError()

src/pip/_internal/distributions/sdist.py

@@ -68,7 +68,10 @@ def _prepare_build_backend(self, finder: PackageFinder) -> None:
 
         self.req.build_env = BuildEnvironment()
         self.req.build_env.install_requirements(
-            finder, pyproject_requires, "overlay", kind="build dependencies"
+            finder,
+            "overlay",
+            requirements=pyproject_requires,
+            kind="build dependencies",
         )
         conflicting, missing = self.req.build_env.check_requirements(
             self.req.requirements_to_check
@@ -120,7 +123,10 @@ def _install_build_reqs(self, finder: PackageFinder) -> None:
         if conflicting:
             self._raise_conflicts("the backend dependencies", conflicting)
         self.req.build_env.install_requirements(
-            finder, missing, "normal", kind="backend dependencies"
+            finder,
+            "normal",
+            requirements=missing,
+            kind="backend dependencies",
         )
 
     def _raise_conflicts(

tests/functional/test_build_env.py

@@ -81,29 +81,9 @@ def run_with_build_env(
 def test_build_env_allow_empty_requirements_install() -> None:
     finder = make_test_finder()
     build_env = BuildEnvironment()
-    for prefix in ("normal", "overlay"):
-        build_env.install_requirements(
-            finder, [], prefix, kind="Installing build dependencies"
-        )
-
-
-def test_build_env_allow_only_one_install(script: PipTestEnvironment) -> None:
-    create_basic_wheel_for_package(script, "foo", "1.0")
-    create_basic_wheel_for_package(script, "bar", "1.0")
-    finder = make_test_finder(find_links=[os.fspath(script.scratch_path)])
-    build_env = BuildEnvironment()
-    for prefix in ("normal", "overlay"):
-        build_env.install_requirements(
-            finder, ["foo"], prefix, kind=f"installing foo in {prefix}"
-        )
-        with pytest.raises(AssertionError):
-            build_env.install_requirements(
-                finder, ["bar"], prefix, kind=f"installing bar in {prefix}"
-            )
-        with pytest.raises(AssertionError):
-            build_env.install_requirements(
-                finder, [], prefix, kind=f"installing in {prefix}"
-            )
+    build_env.install_requirements(
+        finder, requirements=[], kind="empty"
+    )
 
 
 def test_build_env_requirements_check(script: PipTestEnvironment) -> None:
@@ -132,37 +112,40 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None:
     run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal',
-                                       kind='installing foo in normal')
+        build_env.install_requirements(
+            finder, requirements=["foo", "bar==3.0"], kind="foo"
+        )
 
-        r = build_env.check_requirements(['foo', 'bar', 'other'])
-        assert r == (set(), {'other'}), repr(r)
+        r = build_env.check_requirements(["foo", "bar", "other"])
+        assert r == (set(), {"other"}), repr(r)
 
-        r = build_env.check_requirements(['foo>1.0', 'bar==3.0'])
+        r = build_env.check_requirements(["foo>1.0", "bar==3.0"])
         assert r == (set(), set()), repr(r)
 
-        r = build_env.check_requirements(['foo>3.0', 'bar>=2.5'])
-        assert r == ({('foo==2.0', 'foo>3.0')}, set()), repr(r)
+        r = build_env.check_requirements(["foo>3.0", "bar>=2.5"])
+        assert r == ({("foo==2.0", "foo>3.0")}, set()), repr(r)
         """,
     )
 
     run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['foo', 'bar==3.0'], 'normal',
-                                       kind='installing foo in normal')
-        build_env.install_requirements(finder, ['bar==1.0'], 'overlay',
-                                       kind='installing foo in overlay')
+        build_env.install_requirements(
+            finder, requirements=["foo", "bar==3.0"], kind="foo"
+        )
+        build_env.install_requirements(
+            finder, requirements=["1.0"], kind="bar"
+        )
 
-        r = build_env.check_requirements(['foo', 'bar', 'other'])
-        assert r == (set(), {'other'}), repr(r)
+        r = build_env.check_requirements(["foo", "bar", "other"])
+        assert r == (set(), {"other"}), repr(r)
 
-        r = build_env.check_requirements(['foo>1.0', 'bar==3.0'])
-        assert r == ({('bar==1.0', 'bar==3.0')}, set()), repr(r)
+        r = build_env.check_requirements(["foo>1.0", "bar==3.0"])
+        assert r == ({("bar==1.0", "bar==3.0")}, set()), repr(r)
 
-        r = build_env.check_requirements(['foo>3.0', 'bar>=2.5'])
-        assert r == ({('bar==1.0', 'bar>=2.5'), ('foo==2.0', 'foo>3.0')}, \
-            set()), repr(r)
+        r = build_env.check_requirements(["foo>3.0", "bar>=2.5"])
+        assert r == ({("bar==1.0", "bar>=2.5"), ("foo==2.0", "foo>3.0")}, set()), \
+            repr(r)
         """,
     )
 
@@ -187,19 +170,21 @@ def test_build_env_requirements_check(script: PipTestEnvironment) -> None:
     )
 
 
-def test_build_env_overlay_prefix_has_priority(script: PipTestEnvironment) -> None:
+def test_build_env_latest_install_has_priority(script: PipTestEnvironment) -> None:
     create_basic_wheel_for_package(script, "pkg", "2.0")
     create_basic_wheel_for_package(script, "pkg", "4.3")
     result = run_with_build_env(
         script,
         """
-        build_env.install_requirements(finder, ['pkg==2.0'], 'overlay',
-                                       kind='installing pkg==2.0 in overlay')
-        build_env.install_requirements(finder, ['pkg==4.3'], 'normal',
-                                       kind='installing pkg==4.3 in normal')
+        build_env.install_requirements(
+            finder, requirements=["pkg==4.3"], kind="pkg==4.3"
+        )
+        build_env.install_requirements(
+            finder, requirements=["pkg==2.0"], kind="pkg==2.0"
+        )
         """,
         """
-        print(__import__('pkg').__version__)
+        print(__import__("pkg").__version__)
         """,
     )
     assert result.stdout.strip() == "2.0", str(result)
@@ -230,7 +215,7 @@ def test_build_env_isolation(script: PipTestEnvironment) -> None:
     run_with_build_env(
         script,
         "",
-        r"""
+        R"""
         from distutils.sysconfig import get_python_lib
         import sys