From 4b977e48b5b6d898c67f524263b2cbc4245bab5c Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Thu, 30 Jun 2022 13:15:12 -0400 Subject: [PATCH] Vendor latest requirementslib and pip-shims (#5156) * Vendor in latest pip-shims==0.7.1 * Vendor in latest pip-shims 0.7.2 * Vendor in latest requirementslib 1.6.6 --- pipenv/vendor/pip_shims/__init__.py | 2 +- pipenv/vendor/pip_shims/compat.py | 81 +++++---- pipenv/vendor/pip_shims/models.py | 54 ++---- pipenv/vendor/requirementslib/__init__.py | 2 +- .../vendor/requirementslib/models/__init__.py | 1 - .../requirementslib/models/old_pip_utils.py | 163 ++++++++++++++++++ .../requirementslib/models/requirements.py | 2 +- .../requirementslib/models/setup_info.py | 31 +++- pipenv/vendor/vendor.txt | 4 +- 9 files changed, 256 insertions(+), 84 deletions(-) create mode 100644 pipenv/vendor/requirementslib/models/old_pip_utils.py diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 1626a0875a..7575fc8823 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -25,7 +25,7 @@ from . import shims -__version__ = "0.7.0" +__version__ = "0.7.2" if "pip_shims" in sys.modules: diff --git a/pipenv/vendor/pip_shims/compat.py b/pipenv/vendor/pip_shims/compat.py index 882a226ec5..fdc67aed13 100644 --- a/pipenv/vendor/pip_shims/compat.py +++ b/pipenv/vendor/pip_shims/compat.py @@ -48,7 +48,7 @@ Union, ) - from pipenv.patched.notpip._vendor.requests import Session + from requests import Session from .utils import TShim, TShimmedFunc, TShimmedPath @@ -303,24 +303,29 @@ def temp_environ(): @contextlib.contextmanager -def get_requirement_tracker(req_tracker_creator=None): - # type: (Optional[Callable]) -> Generator[Optional[TReqTracker], None, None] - root = os.environ.get("PIP_REQ_TRACKER") - if not req_tracker_creator: +def get_tracker(tracker_creator=None, tracker_type="REQ"): + # type: (Optional[Callable]) -> Generator[Optional[TReqTracker, TBuildTracker], None, None] + env_var = "PIP_REQ_TRACKER" + prefix = "req-tracker" + if tracker_type == "BUILD": # Replaced the req tracker in pip>=22.1 + env_var = "PIP_BUILD_TRACKER" + prefix = "build-tracker" + root = os.environ.get(env_var) + if not tracker_creator: yield None else: req_tracker_args = [] - _, required_args = get_method_args(req_tracker_creator.__init__) # type: ignore + _, required_args = get_method_args(tracker_creator.__init__) # type: ignore with ExitStack() as ctx: if root is None: - root = ctx.enter_context(TemporaryDirectory(prefix="req-tracker")) + root = ctx.enter_context(TemporaryDirectory(prefix=prefix)) if root: root = str(root) ctx.enter_context(temp_environ()) - os.environ["PIP_REQ_TRACKER"] = root + os.environ[env_var] = root if required_args is not None and "root" in required_args: req_tracker_args.append(root) - with req_tracker_creator(*req_tracker_args) as tracker: + with tracker_creator(*req_tracker_args) as tracker: yield tracker @@ -804,6 +809,7 @@ def _ensure_finder( @contextlib.contextmanager def make_preparer( preparer_fn, # type: TShimmedFunc + build_tracker_fn=None, # type: Optional[TShimmedFunc] req_tracker_fn=None, # type: Optional[TShimmedFunc] build_dir=None, # type: Optional[str] src_dir=None, # type: Optional[str] @@ -823,6 +829,7 @@ def make_preparer( install_cmd=None, # type: Optional[TCommandInstance] finder_provider=None, # type: Optional[TShimmedFunc] verbosity=0, # type: Optional[int] + check_build_deps=False, # type: Optional[bool] ): # (...) -> ContextManager """ @@ -890,6 +897,8 @@ def make_preparer( required_args, required_kwargs = get_allowed_args(preparer_fn) # type: ignore if not req_tracker and not req_tracker_fn and "req_tracker" in required_args: raise TypeError("No requirement tracker and no req tracker generator found!") + if not build_tracker and not build_tracker_fn and "build_tracker" in required_args: + raise TypeError("No build tracker and no build tracker generator found!") if "downloader" in required_args and not downloader_provider: raise TypeError("no downloader provided, but one is required to continue!") pip_options_created = options is None @@ -934,19 +943,27 @@ def make_preparer( preparer_args["in_tree_build"] = True if "verbosity" in required_args: preparer_args["verbosity"] = verbosity - req_tracker_fn = resolve_possible_shim(req_tracker_fn) - req_tracker_fn = nullcontext if not req_tracker_fn else req_tracker_fn - with req_tracker_fn() as tracker_ctx: - if "req_tracker" in required_args: + if "check_build_deps" in required_args: + preparer_args["check_build_deps"] = check_build_deps + + if "build_tracker" in required_args: + build_tracker_fn = resolve_possible_shim(build_tracker_fn) + build_tracker_fn = nullcontext if not build_tracker_fn else build_tracker_fn + with build_tracker_fn() as tracker_ctx: + build_tracker = tracker_ctx if build_tracker is None else build_tracker + preparer_args["build_tracker"] = build_tracker + preparer_args["lazy_wheel"] = True + result = call_function_with_correct_args(preparer_fn, **preparer_args) + yield result + if "req_tracker" in required_args: + req_tracker_fn = resolve_possible_shim(req_tracker_fn) + req_tracker_fn = nullcontext if not req_tracker_fn else req_tracker_fn + with req_tracker_fn() as tracker_ctx: req_tracker = tracker_ctx if req_tracker is None else req_tracker preparer_args["req_tracker"] = req_tracker - if "build_tracker" in required_args: - req_tracker = tracker_ctx if req_tracker is None else req_tracker - build_tracker = req_tracker if build_tracker is None else build_tracker - preparer_args["build_tracker"] = build_tracker - preparer_args["lazy_wheel"] = True - result = call_function_with_correct_args(preparer_fn, **preparer_args) - yield result + preparer_args["lazy_wheel"] = True + result = call_function_with_correct_args(preparer_fn, **preparer_args) + yield result @contextlib.contextmanager @@ -1351,14 +1368,6 @@ def resolve( # noqa:C901 wheel_download_dir=wheel_download_dir, **kwargs, ) # type: ignore - if getattr(reqset, "prepare_files", None): - reqset.add_requirement(ireq) - results = reqset.prepare_files(finder) - result = reqset.requirements - reqset.cleanup_files() - return result - if make_preparer_provider is None: - raise TypeError("Cannot create requirement preparer, cannot resolve!") preparer_args = { "build_dir": kwargs["build_dir"], @@ -1401,12 +1410,26 @@ def resolve( # noqa:C901 _, required_resolver_args = get_method_args(resolver.resolve) resolver_args = [] if "requirement_set" in required_resolver_args.args: - reqset.add_requirement(ireq) + if hasattr(reqset, "add_requirement"): + reqset.add_requirement(ireq) + else: # Pip >= 22.1.0 + resolver._add_requirement_to_set(reqset, ireq) resolver_args.append(reqset) elif "root_reqs" in required_resolver_args.args: resolver_args.append([ireq]) if "check_supported_wheels" in required_resolver_args.args: resolver_args.append(check_supported_wheels) + if getattr(reqset, "prepare_files", None): + if hasattr(reqset, "add_requirement"): + reqset.add_requirement(ireq) + else: # Pip >= 22.1.0 + resolver._add_requirement_to_set(reqset, ireq) + results = reqset.prepare_files(finder) + result = reqset.requirements + reqset.cleanup_files() + return result + if make_preparer_provider is None: + raise TypeError("Cannot create requirement preparer, cannot resolve!") result_reqset = resolver.resolve(*resolver_args) # type: ignore if result_reqset is None: result_reqset = reqset diff --git a/pipenv/vendor/pip_shims/models.py b/pipenv/vendor/pip_shims/models.py index 0a344c8437..0ed63c3818 100644 --- a/pipenv/vendor/pip_shims/models.py +++ b/pipenv/vendor/pip_shims/models.py @@ -56,47 +56,6 @@ ) -PIP_VERSION_SET = { - "7.0.0", - "7.0.1", - "7.0.2", - "7.0.3", - "7.1.0", - "7.1.1", - "7.1.2", - "8.0.0", - "8.0.1", - "8.0.2", - "8.0.3", - "8.1.0", - "8.1.1", - "8.1.2", - "9.0.0", - "9.0.1", - "9.0.2", - "9.0.3", - "10.0.0", - "10.0.1", - "18.0", - "18.1", - "19.0", - "19.0.1", - "19.0.2", - "19.0.3", - "19.1", - "19.1.1", - "19.2", - "19.2.1", - "19.2.2", - "19.2.3", - "19.3", - "19.3.1", - "20.0", - "20.0.1", - "20.0.2", -} - - ImportTypesBase = collections.namedtuple( "ImportTypes", ["FUNCTION", "CLASS", "MODULE", "CONTEXTMANAGER"] ) @@ -992,6 +951,9 @@ def import_pip(): RequirementSet = ShimmedPathCollection("RequirementSet", ImportTypes.CLASS) RequirementSet.create_path("req.req_set.RequirementSet", "7.0.0", "9999") +BuildTracker = ShimmedPathCollection("BuildTracker", ImportTypes.CONTEXTMANAGER) +BuildTracker.create_path("operations.build.build_tracker.BuildTracker", "22.1", "9999") + RequirementTracker = ShimmedPathCollection( "RequirementTracker", ImportTypes.CONTEXTMANAGER ) @@ -1021,11 +983,18 @@ def import_pip(): "get_requirement_tracker", ImportTypes.CONTEXTMANAGER ) get_requirement_tracker.set_default( - functools.partial(compat.get_requirement_tracker, RequirementTracker.shim()) + functools.partial(compat.get_tracker, RequirementTracker.shim()) ) get_requirement_tracker.create_path( "req.req_tracker.get_requirement_tracker", "7.0.0", "9999" ) +get_build_tracker = ShimmedPathCollection("get_build_tracker", ImportTypes.CONTEXTMANAGER) +get_build_tracker.set_default( + functools.partial(compat.get_tracker, BuildTracker.shim(), tracker_type="BUILD") +) +get_build_tracker.create_path( + "operations.build.build_tracker.get_build_tracker", "7.0.0", "9999" +) Resolver = ShimmedPathCollection("Resolver", ImportTypes.CLASS) Resolver.create_path("resolve.Resolver", "7.0.0", "19.1.1") @@ -1137,6 +1106,7 @@ def import_pip(): install_cmd_provider=InstallCommand, preparer_fn=RequirementPreparer, downloader_provider=Downloader, + build_tracker_fn=get_build_tracker, req_tracker_fn=get_requirement_tracker, finder_provider=get_package_finder, ) diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index 336fcc9a38..0563a4e4ae 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -10,7 +10,7 @@ from .models.pipfile import Pipfile from .models.requirements import Requirement -__version__ = "1.6.5" +__version__ = "1.6.6" logger = logging.getLogger(__name__) diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index 40a96afc6f..e69de29bb2 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- diff --git a/pipenv/vendor/requirementslib/models/old_pip_utils.py b/pipenv/vendor/requirementslib/models/old_pip_utils.py new file mode 100644 index 0000000000..0f1512b5f9 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/old_pip_utils.py @@ -0,0 +1,163 @@ +"""These were old pip utils that were dropped starting in pip 22.1 but +`requirementslib` still deeply depends upon. + +In the interest of getting the build working with the latest version of +pip again, this workaround to copy these dependent utils in was +provided. Ideally the code that depends on this behavior would be +modernized and refactored to not require it. +""" +import logging +import os +import shutil +import stat + +from pipenv.vendor.vistir.path import rmtree + +logger = logging.getLogger(__name__) + + +try: # Required for pip>=22.1 + from typing import Dict, Iterable, List, Optional + + from pipenv.patched.notpip._internal.models.link import Link + from pipenv.patched.notpip._internal.network.download import Downloader + from pipenv.patched.notpip._internal.operations.prepare import ( + File, + get_file_url, + get_http_url, + unpack_vcs_link, + ) + from pipenv.patched.notpip._internal.utils.hashes import Hashes + from pipenv.patched.notpip._internal.utils.unpacking import unpack_file + + def is_socket(path): + # type: (str) -> bool + return stat.S_ISSOCK(os.lstat(path).st_mode) + + def copy2_fixed(src, dest): + # type: (str, str) -> None + """Wrap shutil.copy2() but map errors copying socket files to + SpecialFileError as expected. + + See also https://bugs.python.org/issue37700. + """ + try: + shutil.copy2(src, dest) + except OSError: + for f in [src, dest]: + try: + is_socket_file = is_socket(f) + except OSError: + # An error has already occurred. Another error here is not + # a problem and we can ignore it. + pass + else: + if is_socket_file: + raise shutil.SpecialFileError( + "`{f}` is a socket".format(**locals()) + ) + + raise + + def _copy2_ignoring_special_files(src: str, dest: str) -> None: + """Copying special files is not supported, but as a convenience to + users we skip errors copying them. + + This supports tools that may create e.g. socket files in the + project source directory. + """ + try: + copy2_fixed(src, dest) + except shutil.SpecialFileError as e: + # SpecialFileError may be raised due to either the source or + # destination. If the destination was the cause then we would actually + # care, but since the destination directory is deleted prior to + # copy we ignore all of them assuming it is caused by the source. + logger.warning( + "Ignoring special file error '%s' encountered copying %s to %s.", + str(e), + src, + dest, + ) + + def _copy_source_tree(source: str, target: str) -> None: + target_abspath = os.path.abspath(target) + target_basename = os.path.basename(target_abspath) + target_dirname = os.path.dirname(target_abspath) + + def ignore(d: str, names: List[str]) -> List[str]: + skipped: List[str] = [] + if d == source: + # Pulling in those directories can potentially be very slow, + # exclude the following directories if they appear in the top + # level dir (and only it). + # See discussion at https://github.com/pypa/pip/pull/6770 + skipped += [".tox", ".nox"] + if os.path.abspath(d) == target_dirname: + # Prevent an infinite recursion if the target is in source. + # This can happen when TMPDIR is set to ${PWD}/... + # and we copy PWD to TMPDIR. + skipped += [target_basename] + return skipped + + shutil.copytree( + source, + target, + ignore=ignore, + symlinks=True, + copy_function=_copy2_ignoring_special_files, + ) + + def old_unpack_url( + link: Link, + location: str, + download: Downloader, + verbosity: int, + download_dir: Optional[str] = None, + hashes: Optional[Hashes] = None, + ) -> Optional[File]: + """Unpack link into location, downloading if required. + + :param hashes: A Hashes object, one of whose embedded hashes must match, + or HashMismatch will be raised. If the Hashes is empty, no matches are + required, and unhashable types of requirements (like VCS ones, which + would ordinarily raise HashUnsupported) are allowed. + """ + # non-editable vcs urls + if link.is_vcs: + unpack_vcs_link(link, location, verbosity=verbosity) + return None + + # Once out-of-tree-builds are no longer supported, could potentially + # replace the below condition with `assert not link.is_existing_dir` + # - unpack_url does not need to be called for in-tree-builds. + # + # As further cleanup, _copy_source_tree and accompanying tests can + # be removed. + # + # TODO when use-deprecated=out-of-tree-build is removed + if link.is_existing_dir(): + if os.path.isdir(location): + rmtree(location) + _copy_source_tree(link.file_path, location) + return None + + # file urls + if link.is_file: + file = get_file_url(link, download_dir, hashes=hashes) + # http urls + else: + file = get_http_url( + link, + download, + download_dir, + hashes=hashes, + ) + # unpack the archive to the build dir location. even when only downloading + # archives, they have to be unpacked to parse dependencies, except wheels + if not link.is_wheel: + unpack_file(file.path, location, file.content_type) + return file + +except ImportError: + raise diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 435bd2a656..33db00da92 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1025,7 +1025,7 @@ def parse_name(self): name, _, _ = name.partition("&") if name is None and self.is_named: name = self._parse_name_from_line() - elif name is None and self.is_file or self.is_remote_url or self.is_path: + elif name is None and (self.is_file or self.is_remote_url or self.is_path): if self.is_local: name = self._parse_name_from_path() if name is not None: diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index b799ff613c..2a3e969cf8 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1565,13 +1565,30 @@ def from_ireq(cls, ireq, subdir=None, finder=None, session=None): } call_function_with_correct_args(build_location_func, **build_kwargs) ireq.ensure_has_source_dir(kwargs["src_dir"]) - pip_shims.shims.shim_unpack( - download_dir=download_dir, - ireq=ireq, - only_download=only_download, - session=session, - hashes=ireq.hashes(False), - ) + try: # Support for pip >= 21.1 + from pipenv.patched.notpip._internal.network.download import Downloader + + from pipenv.vendor.requirementslib.models.old_pip_utils import old_unpack_url + + location = None + if getattr(ireq, "source_dir", None): + location = ireq.source_dir + old_unpack_url( + link=ireq.link, + location=location, + download=Downloader(session, "off"), + verbosity=1, + download_dir=download_dir, + hashes=ireq.hashes(True), + ) + except ImportError: + pip_shims.shims.shim_unpack( + download_dir=download_dir, + ireq=ireq, + only_download=only_download, + session=session, + hashes=ireq.hashes(False), + ) created = cls.create( ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs, stack=stack ) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 07d6633161..5df88e5588 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -19,7 +19,7 @@ packaging==21.3 parse==1.19.0 pep517==0.11.0 pexpect==4.8.0 -pip-shims==0.7.0 +pip-shims==0.7.2 pipdeptree==2.2.1 platformdirs==2.4.0 plette[validation]==0.2.3 @@ -28,7 +28,7 @@ pyparsing==3.0.7 python-dateutil==2.8.2 python-dotenv==0.19.0 pythonfinder==1.2.10 -requirementslib==1.6.5 +requirementslib==1.6.6 shellingham==1.4.0 six==1.16.0 termcolor==1.1.0