FIX Respect yanked flag (#208)

* Respect yanked flag

* changelog

* fix generator

* Update CHANGELOG.md

Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>

* Address comments

* [integration]

---------

Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com>

Author: ryanking13

CHANGELOG.md

@@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### Fixed
+
+- micropip now respects the `yanked` flag in the PyPI Simple API.
+  [#208](https://github.com/pyodide/micropip/pull/208)
+
 ## [0.9.0] - 2024/02/01
 
 ### Fixed

micropip/constants.py

@@ -1,3 +1,10 @@
 FAQ_URLS = {
     "cant_find_wheel": "https://pyodide.org/en/stable/usage/faq.html#why-can-t-micropip-find-a-pure-python-wheel-for-a-package"
 }
+
+# https://github.com/pypa/pip/blob/de44d991024ca8a03e9433ca6178f9a5f661754f/src/pip/_internal/resolution/resolvelib/resolver.py#L164-L167
+YANKED_WARNING_MESSAGE = (
+    "The candidate selected for download or install is a "
+    "yanked version: '%s' candidate (version %s "
+    "at %s)\nReason for being yanked: %s"
+)

micropip/package_index.py

@@ -163,6 +163,11 @@ def _compatible_wheels(
             # Size of the file in bytes, if available (PEP 700)
             # This key is not available in the Simple API HTML response, so this field may be None
             size = file.get("size")
+
+            # PEP-592:
+            # yanked can be an arbitrary string (reason) or bool.
+            yanked_reason = file.get("yanked", False)
+
             yield WheelInfo.from_package_index(
                 name=name,
                 filename=filename,
@@ -171,6 +176,7 @@ def _compatible_wheels(
                 sha256=sha256,
                 size=size,
                 core_metadata=core_metadata,
+                yanked_reason=yanked_reason,
             )
 
     @classmethod

micropip/transaction.py

@@ -2,6 +2,7 @@
 import importlib.metadata
 import logging
 import warnings
+from collections.abc import Iterable
 from dataclasses import dataclass, field
 from importlib.metadata import PackageNotFoundError
 from urllib.parse import urlparse
@@ -19,7 +20,7 @@
     Requirement,
 )
 from ._vendored.packaging.src.packaging.utils import canonicalize_name
-from .constants import FAQ_URLS
+from .constants import FAQ_URLS, YANKED_WARNING_MESSAGE
 from .package import PackageMetadata
 from .package_index import ProjectInfo
 from .wheelinfo import WheelInfo
@@ -238,6 +239,16 @@ async def _add_requirement_from_package_index(self, req: Requirement):
 
         logger.debug("Transaction: Selected wheel: %r", wheel)
 
+        if wheel.yanked:
+            yanked_reason = wheel.yanked_reason if wheel.yanked_reason else "None"
+            logger.info(
+                YANKED_WARNING_MESSAGE,
+                wheel.name,
+                str(wheel.version),
+                wheel.url,
+                yanked_reason,
+            )
+
         # Maybe while we were downloading pypi_json some other branch
         # installed the wheel?
         satisfied, ver = self.check_version_satisfied(req)
@@ -327,6 +338,8 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo:
         reverse=True,
     )
 
+    yanked_versions: list[list[WheelInfo]] = []
+
     for ver in candidate_versions:
         if ver not in releases:
             warnings.warn(
@@ -335,22 +348,43 @@ def find_wheel(metadata: ProjectInfo, req: Requirement) -> WheelInfo:
             )
             continue
 
-        best_wheel = None
-        best_tag_index = float("infinity")
+        wheels = list(releases[ver])
 
-        wheels = releases[ver]
-        for wheel in wheels:
-            tag_index = best_compatible_tag_index(wheel.tags)
-            if tag_index is not None and tag_index < best_tag_index:
-                best_wheel = wheel
-                best_tag_index = tag_index
+        # If the version is yanked, put it in the end of the candidate list.
+        # If we can't find a wheel that satisfies the requirement,
+        # install the yanked version as a last resort.
+        # when the version is yanked, all wheels are yanked, so we can check only the first wheel.
+        yanked = wheels and wheels[0].yanked
+        if yanked:
+            yanked_versions.append(wheels)
+            continue
+
+        best_wheel = _find_best_wheel(wheels)
 
         if best_wheel is not None:
-            return wheel
+            return best_wheel
+
+    for wheels in yanked_versions:
+        best_wheel = _find_best_wheel(wheels)
+
+        if best_wheel is not None:
+            return best_wheel
 
     raise ValueError(
         f"Can't find a pure Python 3 wheel for '{req}'.\n"
         f"See: {FAQ_URLS['cant_find_wheel']}\n"
         "You can use `await micropip.install(..., keep_going=True)` "
         "to get a list of all packages with missing wheels."
     )
+
+
+def _find_best_wheel(wheels: Iterable[WheelInfo]) -> WheelInfo | None:
+    best_wheel = None
+    best_tag_index = float("infinity")
+    for wheel in wheels:
+        tag_index = best_compatible_tag_index(wheel.tags)
+        if tag_index is not None and tag_index < best_tag_index:
+            best_wheel = wheel
+            best_tag_index = tag_index
+
+    return best_wheel

micropip/wheelinfo.py

@@ -44,6 +44,9 @@ class WheelInfo:
     sha256: str | None = None
     size: int | None = None  # Size in bytes, if available (PEP 700)
     core_metadata: DistributionMetadata = None  # Wheel's metadata (PEP 658 / PEP-714)
+    yanked_reason: str | bool = (
+        False  # Whether the wheel has been yanked and the reason (if given) (PEP-592)
+    )
 
     # Fields below are only available after downloading the wheel, i.e. after calling `download()`.
 
@@ -61,6 +64,7 @@ def __post_init__(self):
         ), self.url
         self._project_name = safe_name(self.name)
         self.metadata_url = self.url + ".metadata"
+        self.yanked = bool(self.yanked_reason)
 
     @classmethod
     def from_url(cls, url: str) -> "WheelInfo":
@@ -100,6 +104,7 @@ def from_package_index(
         sha256: str | None,
         size: int | None,
         core_metadata: DistributionMetadata = None,
+        yanked_reason: str | bool = False,
     ) -> "WheelInfo":
         """Extract available metadata from response received from package index"""
         parsed_url = urlparse(url)
@@ -116,6 +121,7 @@ def from_package_index(
             sha256=sha256,
             size=size,
             core_metadata=core_metadata,
+            yanked_reason=yanked_reason,
         )
 
     async def install(self, target: Path) -> None:

tests/integration/test_integration.py

@@ -52,6 +52,26 @@ async def _run(selenium):
     _run(selenium_standalone_micropip)
 
 
+@integration_test_only
+def test_integration_install_yanked(selenium_standalone_micropip, pytestconfig):
+    @run_in_pyodide
+    async def _run(selenium):
+        import contextlib
+        import io
+
+        import micropip
+
+        with io.StringIO() as buf, contextlib.redirect_stdout(buf):
+            # install yanked version
+            await micropip.install("black==21.11b0", verbose=True)
+
+            captured = buf.getvalue()
+            assert "The candidate selected for download or install is a" in captured
+            assert "'black' candidate (version 21.11b0" in captured
+
+    _run(selenium_standalone_micropip)
+
+
 @integration_test_only
 def test_integration_list_basic(selenium_standalone_micropip, pytestconfig):
     @run_in_pyodide

tests/test_transaction.py

@@ -215,6 +215,47 @@ def test_find_wheel_invalid_version():
     assert str(wheel.version) == "0.15.5"
 
 
+def test_yanked_version():
+    from micropip._vendored.packaging.src.packaging.requirements import Requirement
+    from micropip.transaction import find_wheel
+
+    versions = ["0.0.1", "0.15.5", "0.9.1"]
+
+    # Mark 0.15.5 as yanked
+    # convert generator --> list and monkeypatch the yanked value
+    metadata = _pypi_metadata("dummy_module", {v: ["py3"] for v in versions})
+    for version in list(metadata.releases):
+        wheels = list(metadata.releases[version])
+        for wheel in wheels:
+            if str(wheel.version) == "0.15.5":
+                wheel.yanked = True
+
+        metadata.releases[version] = wheels
+
+    # case 1: yanked version should be skipped and the next best version should be selected
+
+    requirement1 = Requirement("dummy_module")
+    wheel = find_wheel(metadata, requirement1)
+
+    assert str(wheel.version) == "0.9.1"
+
+    # case 2: yanked version is explicitly requested, so it should be selected
+
+    requirement2 = Requirement("dummy_module==0.15.5")
+    wheel = find_wheel(metadata, requirement2)
+
+    assert str(wheel.version) == "0.15.5"
+
+    # case 3: yanked version is not explicitly requested, but it is the only version available
+    # so it should be selected
+
+    requirement3 = Requirement("dummy_module>0.10.0")
+
+    wheel = find_wheel(metadata, requirement3)
+
+    assert str(wheel.version) == "0.15.5"
+
+
 _best_tag_test_cases = (
     "package, version, incompatible_tags, compatible_tags",
     # Tests assume that `compatible_tags` is sorted from least to most compatible: