Merge pull request #2643 from pypa/fix-find-links

Fix resolution using `dependency_links` with ssh

Author: uranusjr

news/2434.bugfix

@@ -0,0 +1 @@
+Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled.

news/2643.bugfix

@@ -0,0 +1 @@
+Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``.

news/2643.feature

@@ -0,0 +1 @@
+Enhanced resolution of editable and VCS dependencies.

pipenv/patched/piptools/repositories/pypi.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
-
+import copy
 import hashlib
 import os
 import sys
@@ -21,18 +21,16 @@
     SafeFileCache,
 )
 
-from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
-from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
-from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
-from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
-from pipenv.patched.notpip._vendor.pyparsing import ParseException
+from pipenv.patched.notpip._vendor.packaging.requirements import Requirement
+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier
+from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable
 from pipenv.patched.notpip._internal.exceptions import InstallationError
+from pipenv.patched.notpip._internal.vcs import VcsSupport
 
-from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
-                     make_install_requirement, format_requirement, dedup, clean_requires_python)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table,
+                     make_install_requirement, clean_requires_python)
 
 from .base import BaseRepository
 
@@ -64,15 +62,20 @@ def __init__(self, *args, **kwargs):
     def get_hash(self, location):
         # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
         hash_value = None
-        can_hash = location.hash
+        vcs = VcsSupport()
+        orig_scheme = location.scheme
+        new_location = copy.deepcopy(location)
+        if orig_scheme in vcs.all_schemes:
+            new_location.url = new_location.url.split("+", 1)[-1]
+        can_hash = new_location.hash
         if can_hash:
             # hash url WITH fragment
-            hash_value = self.get(location.url)
+            hash_value = self.get(new_location.url)
         if not hash_value:
-            hash_value = self._get_file_hash(location)
+            hash_value = self._get_file_hash(new_location)
             hash_value = hash_value.encode('utf8')
         if can_hash:
-            self.set(location.url, hash_value)
+            self.set(new_location.url, hash_value)
         return hash_value.decode('utf8')
 
     def _get_file_hash(self, location):
@@ -276,6 +279,13 @@ def get_legacy_dependencies(self, ireq):
             setup_requires = {}
             dist = None
             if ireq.editable:
+                try:
+                    from pipenv.utils import chdir
+                    with chdir(ireq.setup_py_dir):
+                        from setuptools.dist import distutils
+                        distutils.core.run_setup(ireq.setup_py)
+                except (ImportError, InstallationError, TypeError, AttributeError):
+                    pass
                 try:
                     dist = ireq.get_dist()
                 except InstallationError:
@@ -425,6 +435,10 @@ def get_hashes(self, ireq):
         if ireq.editable:
             return set()
 
+        vcs = VcsSupport()
+        if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
+            return set()
+
         if not is_pinned_requirement(ireq):
             raise TypeError(
                 "Expected pinned requirement, got {}".format(ireq))

pipenv/utils.py

@@ -1361,3 +1361,19 @@ def is_virtual_environment(path):
             if python_like.is_file() and os.access(str(python_like), os.X_OK):
                 return True
     return False
+
+
+@contextmanager
+def chdir(path):
+    """Context manager to change working directories."""
+    from ._compat import Path
+    if not path:
+        return
+    prev_cwd = Path.cwd().as_posix()
+    if isinstance(path, Path):
+        path = path.as_posix()
+    os.chdir(str(path))
+    try:
+        yield
+    finally:
+        os.chdir(prev_cwd)

tasks/vendoring/patches/patched/piptools.patch

@@ -19,18 +19,22 @@ index 4e6174c..75f9b49 100644
  # NOTE
  # We used to store the cache dir under ~/.pip-tools, which is not the
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index 1c4b943..c922be1 100644
+index 1c4b943..91902dc 100644
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
-@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
- 
+@@ -1,9 +1,10 @@
+ # coding: utf-8
+ from __future__ import (absolute_import, division, print_function,
+                         unicode_literals)
+-
++import copy
  import hashlib
  import os
 +import sys
  from contextlib import contextmanager
  from shutil import rmtree
 
-@@ -15,13 +16,24 @@ from .._compat import (
+@@ -15,13 +16,22 @@ from .._compat import (
      Wheel,
      FAVORITE_HASH,
      TemporaryDirectory,
@@ -40,25 +44,23 @@ index 1c4b943..c922be1 100644
 +    SafeFileCache,
  )
 
-+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
-+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
-+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
-+from pip._vendor.packaging.markers import Marker, Op, Value, Variable
-+from pip._vendor.pyparsing import ParseException
+-from ..cache import CACHE_DIR
++from pip._vendor.packaging.requirements import Requirement
++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier
++from pip._vendor.packaging.markers import Op, Value, Variable
 +from pip._internal.exceptions import InstallationError
++from pip._internal.vcs import VcsSupport
 +
- from ..cache import CACHE_DIR
 +from pipenv.environments import PIPENV_CACHE_DIR
  from ..exceptions import NoCandidateFound
--from ..utils import (fs_str, is_pinned_requirement, lookup_table,
+ from ..utils import (fs_str, is_pinned_requirement, lookup_table,
 -                     make_install_requirement)
-+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
-+                     make_install_requirement, format_requirement, dedup, clean_requires_python)
++                     make_install_requirement, clean_requires_python)
 +
  from .base import BaseRepository
 
 
-@@ -37,6 +49,40 @@ except ImportError:
+@@ -37,6 +47,45 @@ except ImportError:
      from pip.wheel import WheelCache
 
 
@@ -77,15 +79,20 @@ index 1c4b943..c922be1 100644
 +    def get_hash(self, location):
 +        # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
 +        hash_value = None
-+        can_hash = location.hash
++        vcs = VcsSupport()
++        orig_scheme = location.scheme
++        new_location = copy.deepcopy(location)
++        if orig_scheme in vcs.all_schemes:
++            new_location.url = new_location.url.split("+", 1)[-1]
++        can_hash = new_location.hash
 +        if can_hash:
 +            # hash url WITH fragment
-+            hash_value = self.get(location.url)
++            hash_value = self.get(new_location.url)
 +        if not hash_value:
-+            hash_value = self._get_file_hash(location)
++            hash_value = self._get_file_hash(new_location)
 +            hash_value = hash_value.encode('utf8')
 +        if can_hash:
-+            self.set(location.url, hash_value)
++            self.set(new_location.url, hash_value)
 +        return hash_value.decode('utf8')
 +
 +    def _get_file_hash(self, location):
@@ -99,7 +106,7 @@ index 1c4b943..c922be1 100644
  class PyPIRepository(BaseRepository):
      DEFAULT_INDEX_URL = PyPI.simple_url
 
-@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository):
+@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository):
      config), but any other PyPI mirror can be used if index_urls is
      changed/configured on the Finder.
      """
@@ -113,7 +120,7 @@ index 1c4b943..c922be1 100644
 
          index_urls = [pip_options.index_url] + pip_options.extra_index_urls
          if pip_options.no_index:
-@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository):
+@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository):
          # of all secondary dependencies for the given requirement, so we
          # only have to go to disk once for each requirement
          self._dependencies_cache = {}
@@ -131,7 +138,7 @@ index 1c4b943..c922be1 100644
 
      def freshen_build_caches(self):
          """
-@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository):
+@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository):
          if ireq.editable:
              return ireq  # return itself as the best match
 
@@ -148,7 +155,7 @@ index 1c4b943..c922be1 100644
 
          # Reuses pip's internal candidate sort key to sort
          matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
-@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository):
+@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository):
          best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
 
          # Turn the candidate into a pinned InstallRequirement
@@ -223,7 +230,7 @@ index 1c4b943..c922be1 100644
          """
          Given a pinned or an editable InstallRequirement, returns a set of
          dependencies (also InstallRequirements, but not necessarily pinned).
-@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository):
+@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository):
                      os.makedirs(download_dir)
              if not os.path.isdir(self._wheel_download_dir):
                  os.makedirs(self._wheel_download_dir)
@@ -235,6 +242,13 @@ index 1c4b943..c922be1 100644
 +            dist = None
 +            if ireq.editable:
 +                try:
++                    from pipenv.utils import chdir
++                    with chdir(ireq.setup_py_dir):
++                        from setuptools.dist import distutils
++                        distutils.core.run_setup(ireq.setup_py)
++                except (ImportError, InstallationError, TypeError, AttributeError):
++                    pass
++                try:
 +                    dist = ireq.get_dist()
 +                except InstallationError:
 +                    ireq.run_egg_info()
@@ -268,7 +282,7 @@ index 1c4b943..c922be1 100644
                  )
              except TypeError:
                  # Pip >= 10 (new resolver!)
-@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository):
+@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository):
                      finder=self.finder,
                      session=self.session,
                      upgrade_strategy="to-satisfy-only",
@@ -369,7 +383,18 @@ index 1c4b943..c922be1 100644
          return set(self._dependencies_cache[ireq])
 
      def get_hashes(self, ireq):
-@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository):
+@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository):
+         if ireq.editable:
+             return set()
+ 
++        vcs = VcsSupport()
++        if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
++            return set()
++
+         if not is_pinned_requirement(ireq):
+             raise TypeError(
+                 "Expected pinned requirement, got {}".format(ireq))
+@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository):
          # We need to get all of the candidates that match our current version
          # pin, these will represent all of the files that could possibly
          # satisfy this constraint.