macOS Python 3 Framework support (#1711)

Signed-off-by: Bernat Gabor <bgabor8@bloomberg.net>

Author: gaborbernat

setup.cfg

@@ -69,6 +69,7 @@ virtualenv.create =
     cpython3-win = virtualenv.create.via_global_ref.builtin.cpython.cpython3:CPython3Windows
     cpython2-posix = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Posix
     cpython2-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython2macOsFramework
+    cpython3-mac-framework = virtualenv.create.via_global_ref.builtin.cpython.mac_os:CPython3macOsFramework
     cpython2-win = virtualenv.create.via_global_ref.builtin.cpython.cpython2:CPython2Windows
     pypy2-posix = virtualenv.create.via_global_ref.builtin.pypy.pypy2:PyPy2Posix
     pypy2-win = virtualenv.create.via_global_ref.builtin.pypy.pypy2:Pypy2Windows

src/virtualenv/create/via_global_ref/builtin/cpython/common.py

@@ -46,3 +46,11 @@ def _executables(cls, interpreter):
         # for more info on pythonw.exe see https://stackoverflow.com/a/30313091
         python_w = host.parent / "pythonw.exe"
         yield python_w, [python_w.name]
+
+
+def is_mac_os_framework(interpreter):
+    if interpreter.platform == "darwin":
+        framework_var = interpreter.sysconfig_vars.get("PYTHONFRAMEWORK")
+        value = "Python3" if interpreter.version_info.major == 3 else "Python"
+        return framework_var == value
+    return False

src/virtualenv/create/via_global_ref/builtin/cpython/cpython2.py

@@ -9,7 +9,7 @@
 from virtualenv.util.path import Path
 
 from ..python2.python2 import Python2
-from .common import CPython, CPythonPosix, CPythonWindows
+from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework
 
 
 @add_metaclass(abc.ABCMeta)
@@ -50,11 +50,6 @@ def ensure_directories(self):
         return dirs
 
 
-def is_mac_os_framework(interpreter):
-    framework = bool(interpreter.sysconfig_vars.get("PYTHONFRAMEWORK"))
-    return framework and interpreter.platform == "darwin"
-
-
 class CPython2Posix(CPython2, CPythonPosix):
     """CPython 2 on POSIX (excluding macOs framework builds)"""
 

src/virtualenv/create/via_global_ref/builtin/cpython/cpython3.py

@@ -8,7 +8,7 @@
 from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
 from virtualenv.util.path import Path
 
-from .common import CPython, CPythonPosix, CPythonWindows
+from .common import CPython, CPythonPosix, CPythonWindows, is_mac_os_framework
 
 
 @add_metaclass(abc.ABCMeta)
@@ -17,7 +17,9 @@ class CPython3(CPython, Python3Supports):
 
 
 class CPython3Posix(CPythonPosix, CPython3):
-    """"""
+    @classmethod
+    def can_describe(cls, interpreter):
+        return is_mac_os_framework(interpreter) is False and super(CPython3Posix, cls).can_describe(interpreter)
 
 
 class CPython3Windows(CPythonWindows, CPython3):

src/virtualenv/create/via_global_ref/builtin/cpython/mac_os.py

@@ -4,59 +4,89 @@
 import os
 import struct
 import subprocess
+from abc import ABCMeta, abstractmethod
 from textwrap import dedent
 
-from virtualenv.create.via_global_ref.builtin.cpython.common import CPythonPosix
-from virtualenv.create.via_global_ref.builtin.ref import PathRefToDest
+from six import add_metaclass
+
+from virtualenv.create.via_global_ref.builtin.ref import ExePathRefToDest, PathRefToDest
 from virtualenv.util.path import Path
 from virtualenv.util.six import ensure_text
 
-from .cpython2 import CPython2, is_mac_os_framework
+from .common import CPython, CPythonPosix, is_mac_os_framework
+from .cpython2 import CPython2
+from .cpython3 import CPython3
 
 
-class CPython2macOsFramework(CPython2, CPythonPosix):
+@add_metaclass(ABCMeta)
+class CPythonmacOsFramework(CPython):
     @classmethod
     def can_describe(cls, interpreter):
-        return is_mac_os_framework(interpreter) and super(CPython2macOsFramework, cls).can_describe(interpreter)
-
-    def create(self):
-        super(CPython2macOsFramework, self).create()
-
-        # change the install_name of the copied python executable
-        current = os.path.join(self.interpreter.prefix, "Python")
-        fix_mach_o(str(self.exe), current, "@executable_path/../.Python", self.interpreter.max_size)
+        return is_mac_os_framework(interpreter) and super(CPythonmacOsFramework, cls).can_describe(interpreter)
 
     @classmethod
     def sources(cls, interpreter):
-        for src in super(CPython2macOsFramework, cls).sources(interpreter):
+        for src in super(CPythonmacOsFramework, cls).sources(interpreter):
             yield src
-
-        # landmark for exec_prefix
-        name = "lib-dynload"
-        yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib)
-
-        # this must symlink to the host prefix Python
-        marker = Path(interpreter.prefix) / "Python"
-        ref = PathRefToDest(marker, dest=lambda self, _: self.dest / ".Python", must_symlink=True)
+        # add a symlink to the host python image
+        ref = PathRefToDest(cls.image_ref(interpreter), dest=lambda self, _: self.dest / ".Python", must_symlink=True)
         yield ref
 
+    def create(self):
+        super(CPythonmacOsFramework, self).create()
+
+        # change the install_name of the copied python executables
+        target = "@executable_path/../.Python"
+        current = self.current_mach_o_image_path()
+        for src in self._sources:
+            if isinstance(src, ExePathRefToDest):
+                if src.must_copy or not self.symlinks:
+                    exes = [self.bin_dir / src.base]
+                    if not self.symlinks:
+                        exes.extend(self.bin_dir / a for a in src.aliases)
+                    for exe in exes:
+                        fix_mach_o(str(exe), current, target, self.interpreter.max_size)
+
     @classmethod
     def _executables(cls, interpreter):
-        for _, targets in super(CPython2macOsFramework, cls)._executables(interpreter):
+        for _, targets in super(CPythonmacOsFramework, cls)._executables(interpreter):
             # Make sure we use the embedded interpreter inside the framework, even if sys.executable points to the
             # stub executable in ${sys.prefix}/bin.
             # See http://groups.google.com/group/python-virtualenv/browse_thread/thread/17cab2f85da75951
             fixed_host_exe = Path(interpreter.prefix) / "Resources" / "Python.app" / "Contents" / "MacOS" / "Python"
             yield fixed_host_exe, targets
 
+    @abstractmethod
+    def current_mach_o_image_path(self):
+        raise NotImplementedError
+
+    @classmethod
+    def image_ref(cls, interpreter):
+        raise NotImplementedError
+
+
+class CPython2macOsFramework(CPythonmacOsFramework, CPython2, CPythonPosix):
+    @classmethod
+    def image_ref(cls, interpreter):
+        return Path(interpreter.prefix) / "Python"
+
+    def current_mach_o_image_path(self):
+        return os.path.join(self.interpreter.prefix, "Python")
+
+    @classmethod
+    def sources(cls, interpreter):
+        for src in super(CPython2macOsFramework, cls).sources(interpreter):
+            yield src
+        name = "lib-dynload"  # landmark for exec_prefix
+        yield PathRefToDest(interpreter.stdlib_path(name), dest=cls.to_stdlib)
+
     @property
     def reload_code(self):
         result = super(CPython2macOsFramework, self).reload_code
         result = dedent(
             """
         # the bundled site.py always adds the global site package if we're on python framework build, escape this
         import sysconfig
-
         config = sysconfig.get_config_vars()
         before = config["PYTHONFRAMEWORK"]
         try:
@@ -71,6 +101,34 @@ def reload_code(self):
         return result
 
 
+class CPython3macOsFramework(CPythonmacOsFramework, CPython3, CPythonPosix):
+    @classmethod
+    def image_ref(cls, interpreter):
+        return Path(interpreter.prefix) / "Python3"
+
+    def current_mach_o_image_path(self):
+        return "@executable_path/../../../../Python3"
+
+    @property
+    def reload_code(self):
+        result = super(CPython3macOsFramework, self).reload_code
+        result = dedent(
+            """
+        # the bundled site.py always adds the global site package if we're on python framework build, escape this
+        import sys
+        before = sys._framework
+        try:
+            sys._framework = None
+            {}
+        finally:
+            sys._framework = before
+        """.format(
+                result
+            )
+        )
+        return result
+
+
 def fix_mach_o(exe, current, new, max_size):
     """
     https://en.wikipedia.org/wiki/Mach-O

src/virtualenv/discovery/py_info.py

@@ -112,6 +112,8 @@ def _fast_get_system_executable(self):
     def _distutils_install():
         # follow https://github.com/pypa/pip/blob/master/src/pip/_internal/locations.py#L95
         d = Distribution({"script_args": "--no-user-cfg"})  # configuration files not parsed so they do not hijack paths
+        if hasattr(sys, "_framework"):
+            sys._framework = None  # disable macOS static paths for framework
         i = d.get_command_obj("install", create=True)
         i.prefix = os.sep  # paths generated are relative to prefix that contains the path sep, this makes it relative
         i.finalize_options()

tests/unit/create/test_creator.py

@@ -296,7 +296,7 @@ def cross_python(is_inside_ci, session_app_data):
 
 
 @pytest.mark.slow
-def test_cross_major(cross_python, coverage_env, tmp_path, current_fastest, session_app_data):
+def test_cross_major(cross_python, coverage_env, tmp_path, session_app_data, current_fastest):
     cmd = [
         "-v",
         "-v",
@@ -307,8 +307,6 @@ def test_cross_major(cross_python, coverage_env, tmp_path, current_fastest, sess
         "--no-wheel",
         "--activators",
         "",
-        "--creator",
-        current_fastest,
     ]
     result = cli_run(cmd)
     pip_scripts = {i.name.replace(".exe", "") for i in result.creator.script_dir.iterdir() if i.name.startswith("pip")}