From 3d678257b7ccea706e8d4df338530db196fc36c6 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 23 Dec 2021 10:49:17 +0800 Subject: [PATCH 1/4] Fix the index parsing --- pipenv/core.py | 10 ++++-- pipenv/resolver.py | 2 +- pipenv/utils.py | 48 ++++++++++--------------- tests/integration/test_install_basic.py | 13 ++++--- tests/unit/test_utils.py | 23 ++++++++++++ 5 files changed, 58 insertions(+), 38 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 52134bf6d7..6ce17bf315 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -163,9 +163,13 @@ def import_requirements(project, r=None, dev=False): trusted_hosts = [] # Find and add extra indexes. for line in contents.split("\n"): - line_indexes, _trusted_hosts, _ = parse_indexes(line.strip()) - indexes.extend(line_indexes) - trusted_hosts.extend(_trusted_hosts) + index, extra_index, trusted_host, _ = parse_indexes(line.strip(), strict=True) + if index: + indexes = [index] + if extra_index: + indexes.append(extra_index) + if trusted_host: + trusted_hosts.append(trusted_host) indexes = sorted(set(indexes)) trusted_hosts = sorted(set(trusted_hosts)) reqs = [install_req_from_parsed_requirement(f) for f in parse_requirements(r, session=pip_requests)] diff --git a/pipenv/resolver.py b/pipenv/resolver.py index 6ebbe70430..b4c01ad404 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -649,7 +649,7 @@ def parse_packages(packages, pre, clear, system, requirements_dir=None): from pipenv.utils import parse_indexes parsed_packages = [] for package in packages: - indexes, trusted_hosts, line = parse_indexes(package) + *_, line = parse_indexes(package) line = " ".join(line) pf = dict() req = Requirement.from_line(line) diff --git a/pipenv/utils.py b/pipenv/utils.py index 7f0da5412c..1158116731 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -521,10 +521,7 @@ def parse_line( if project is None: from .project import Project project = Project() - url = None - indexes, trusted_hosts, remainder = parse_indexes(line) - if indexes: - url = indexes[0] + index, extra_index, trust_host, remainder = parse_indexes(line) line = " ".join(remainder) req = None # type: Requirement try: @@ -539,10 +536,10 @@ def parse_line( raise ResolutionFailure(f"Failed to resolve requirement from line: {line!s}") else: raise ResolutionFailure(f"Failed to resolve requirement from line: {line!s}") - if url: + if index: try: index_lookup[req.normalized_name] = project.get_source( - url=url, refresh=True).get("name") + url=index, refresh=True).get("name") except TypeError: pass try: @@ -556,12 +553,6 @@ def parse_line( markers_lookup[req.normalized_name] = req.markers.replace('"', "'") return req, index_lookup, markers_lookup - @classmethod - def get_deps_from_line(cls, line): - # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] - req, _, _ = cls.parse_line(line) - return cls.get_deps_from_req(req) - @classmethod def get_deps_from_req(cls, req, resolver=None, resolve_vcs=True): # type: (Requirement, Optional["Resolver"], bool) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] @@ -725,7 +716,7 @@ def pip_command(self): self._pip_command = self._get_pip_command() return self._pip_command - def prepare_pip_args(self, use_pep517=False, build_isolation=True): + def prepare_pip_args(self, use_pep517=None, build_isolation=True): pip_args = [] if self.sources: pip_args = prepare_pip_source_args(self.sources, pip_args) @@ -839,7 +830,6 @@ def get_resolver(self, clear=False): ) with global_tempdir_manager(), get_requirement_tracker() as req_tracker, TemporaryDirectory(suffix="-build", prefix="pipenv-") as directory: - os.environ["PIP_USE_PEP517"] = "false" pip_options = self.pip_options finder = self.finder wheel_cache = WheelCache(pip_options.cache_dir, pip_options.format_control) @@ -2058,24 +2048,22 @@ def looks_like_dir(path): return any(sep in path for sep in seps) -def parse_indexes(line): +def parse_indexes(line, strict=False): from argparse import ArgumentParser - parser = ArgumentParser("indexes") - parser.add_argument( - "--index", "-i", "--index-url", - metavar="index_url", action="store", nargs="?", - ) - parser.add_argument( - "--extra-index-url", "--extra-index", - metavar="extra_indexes", action="append", - ) - parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") + line = line.split("#")[0].strip() + parser = ArgumentParser("indexes", exit_on_error=False) + parser.add_argument("-i", "--index-url", dest="index") + parser.add_argument("--extra-index-url", dest="extra_index") + parser.add_argument("--trusted-host", dest="trusted_host") args, remainder = parser.parse_known_args(line.split()) - index = [] if not args.index else [args.index] - extra_indexes = [] if not args.extra_index_url else args.extra_index_url - indexes = index + extra_indexes - trusted_hosts = args.trusted_host if args.trusted_host else [] - return indexes, trusted_hosts, remainder + index = args.index + extra_index = args.extra_index + trusted_host = args.trusted_host + if strict and sum( + bool(arg) for arg in (index, extra_index, trusted_host, remainder) + ) > 1: + raise ValueError("Index arguments must be on their own lines.") + return index, extra_index, trusted_host, remainder @contextmanager diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f30c792a87..fb1241422d 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -285,18 +285,23 @@ def test_requirements_to_pipfile(PipenvInstance, pypi): # Write a requirements file with open("requirements.txt", "w") as f: - f.write(f"-i {pypi.url}\nrequests[socks]==2.19.1\n") + f.write( + f"-i {pypi.url}\n" + "# -i https://private.pypi.org/simple\n" + "requests[socks]==2.19.1\n" + ) c = p.pipenv("install") assert c.returncode == 0 print(c.stdout) print(c.stderr) - print(subprocess_run(["ls", "-l"]).stdout) - # assert stuff in pipfile assert "requests" in p.pipfile["packages"] assert "extras" in p.pipfile["packages"]["requests"] - + assert not any( + source['url'] == 'https://private.pypi.org/simple' + for source in p.pipfile['source'] + ) # assert stuff in lockfile assert "requests" in p.lockfile["default"] assert "chardet" in p.lockfile["default"] diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 47a755ff81..ec35502881 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -137,6 +137,29 @@ def test_convert_deps_to_pip_unicode(): assert deps[0] == "django==1.10" +@pytest.mark.parametrize("line,result", [ + ("-i https://example.com/simple/", ("https://example.com/simple/", None, None, [])), + ("--extra-index-url=https://example.com/simple/", (None, "https://example.com/simple/", None, [])), + ("--trusted-host=example.com", (None, None, "example.com", [])), + ("# -i https://example.com/simple/", (None, None, None, [])), + ("requests", (None, None, None, ["requests"])) +]) +@pytest.mark.utils +def test_parse_indexes(line, result): + assert pipenv.utils.parse_indexes(line) == result + + +@pytest.mark.parametrize("line", [ + "-i https://example.com/simple/ --extra-index-url=https://extra.com/simple/", + "--extra-index-url https://example.com/simple/ --trusted-host=example.com", + "requests -i https://example.com/simple/", +]) +@pytest.mark.utils +def test_parse_indexes_individual_lines(line): + with pytest.raises(ValueError): + pipenv.utils.parse_indexes(line, strict=True) + + class TestUtils: """Test utility functions in pipenv""" From 32146365bf3efd7493fbcf69b92b995fa4fbcea1 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 23 Dec 2021 11:27:23 +0800 Subject: [PATCH 2/4] remove the useless option --- news/4899.bugfix.rst | 1 + pipenv/utils.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 news/4899.bugfix.rst diff --git a/news/4899.bugfix.rst b/news/4899.bugfix.rst new file mode 100644 index 0000000000..bc61835d0a --- /dev/null +++ b/news/4899.bugfix.rst @@ -0,0 +1 @@ +Fix the index parsing to reject illegal requirements.txt. diff --git a/pipenv/utils.py b/pipenv/utils.py index 1158116731..bd4caf0637 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -2051,7 +2051,7 @@ def looks_like_dir(path): def parse_indexes(line, strict=False): from argparse import ArgumentParser line = line.split("#")[0].strip() - parser = ArgumentParser("indexes", exit_on_error=False) + parser = ArgumentParser("indexes") parser.add_argument("-i", "--index-url", dest="index") parser.add_argument("--extra-index-url", dest="extra_index") parser.add_argument("--trusted-host", dest="trusted_host") From a30ec71254bd2465d62d3b27336aa4be092f7722 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 23 Dec 2021 12:53:42 +0800 Subject: [PATCH 3/4] fix comment ignorance --- pipenv/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index bd4caf0637..cb2cecd1d0 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -2050,7 +2050,9 @@ def looks_like_dir(path): def parse_indexes(line, strict=False): from argparse import ArgumentParser - line = line.split("#")[0].strip() + + comment_re = re.compile(r"(?:^|\s+)#.*$") + line = comment_re.sub("", line) parser = ArgumentParser("indexes") parser.add_argument("-i", "--index-url", dest="index") parser.add_argument("--extra-index-url", dest="extra_index") From 33cbabe20cfda8cd7c3e62fc9731aa7776db7566 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Thu, 23 Dec 2021 15:20:11 +0800 Subject: [PATCH 4/4] fix pip location --- pipenv/patched/notpip/_internal/build_env.py | 2 +- .../patches/patched/_post_pip_import.patch | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 05457c5a00..d8c66b3ff5 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -17,7 +17,7 @@ from pipenv.patched.notpip._vendor.packaging.requirements import Requirement from pipenv.patched.notpip._vendor.packaging.version import Version -from pipenv.patched.notpip import __file__ as pip_location +from pip import __file__ as pip_location from pipenv.patched.notpip._internal.cli.spinners import open_spinner from pipenv.patched.notpip._internal.locations import get_platlib, get_prefixed_libs, get_purelib from pipenv.patched.notpip._internal.metadata import get_environment diff --git a/tasks/vendoring/patches/patched/_post_pip_import.patch b/tasks/vendoring/patches/patched/_post_pip_import.patch index 4a9756ad80..26cd89af3c 100644 --- a/tasks/vendoring/patches/patched/_post_pip_import.patch +++ b/tasks/vendoring/patches/patched/_post_pip_import.patch @@ -26,3 +26,16 @@ index 0ba06c52..6fdb59b7 100644 class LinkCandidate(_InstallRequirementBackedCandidate): +diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py +index 05457c5a..d8c66b3f 100644 +--- a/pipenv/patched/notpip/_internal/build_env.py ++++ b/pipenv/patched/notpip/_internal/build_env.py +@@ -17,7 +17,7 @@ from pipenv.patched.notpip._vendor.certifi import where + from pipenv.patched.notpip._vendor.packaging.requirements import Requirement + from pipenv.patched.notpip._vendor.packaging.version import Version + +-from pipenv.patched.notpip import __file__ as pip_location ++from pip import __file__ as pip_location + from pipenv.patched.notpip._internal.cli.spinners import open_spinner + from pipenv.patched.notpip._internal.locations import get_platlib, get_prefixed_libs, get_purelib + from pipenv.patched.notpip._internal.metadata import get_environment