Vendor pypa/packaging into micropip (#178)

Author: agriyakhetarpal

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        pyodide-version: ['0.27.0a2']
+        pyodide-version: ["0.27.0a2"]
         test-config: [
           # FIXME: recent version of chrome gets timeout
           {runner: selenium, runtime: chrome, runtime-version: "125" },
@@ -28,6 +28,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:
@@ -88,6 +90,7 @@ jobs:
       - uses: actions/checkout@cbb722410c2e876e24abbe8de2cc27693e501dcb # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+          submodules: recursive
 
       - id: check-integration-test-trigger
         name: Check integration test trigger
@@ -110,15 +113,25 @@ jobs:
     environment: PyPi-deploy
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:
           python-version: 3.12
+
+      # IMPORTANT: always build sdist, and then the wheel from
+      # the sdist (like it is currently done here). This is
+      # because we want to ensure that no extra files get
+      # copied, which can be the case with building in-tree.
+      # The MANIFEST.in file ensures that the sdist doesn't
+      # contain any unnecessary files.
       - name: Install requirements and build wheel
         shell: bash -l {0}
         run: |
           python -m pip install build twine
           python -m build .
+
       - name: Publish package
         uses: pypa/gh-action-pypi-publish@release/v1
         with:

.github/workflows/remote_package_index_test.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
 
       - uses: actions/setup-python@v5
         with:

.gitmodules

@@ -0,0 +1,5 @@
+# d8e3b31b734926ebbcaff654279f6855a73e052f, for 24.2 release
+# https://github.com/pypa/packaging/releases/tag/24.2
+[submodule "micropip/_vendored/packaging"]
+	path = micropip/_vendored/packaging
+	url = https://github.com/pypa/packaging/

CHANGELOG.md

@@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (the ones that starts with `../` or `./`)
   [#174](https://github.com/pyodide/micropip/pull/174)
 
+### Added
+
+- `micropip` now vendors `pypa/packaging` for better reliability.
+  [#178](https://github.com/pyodide/micropip/pull/178)
+
 ## [0.8.0] - 2024/12/15
 
 ### Added

MANIFEST.in

@@ -0,0 +1,16 @@
+exclude micropip/_vendored/packaging/.github/*
+exclude micropip/_vendored/packaging/docs/*
+exclude micropip/_vendored/packaging/tasks/*
+exclude micropip/_vendored/packaging/tests/*
+exclude micropip/_vendored/packaging/.pre-commit-config.yaml
+exclude micropip/_vendored/packaging/.readthedocs.yml
+exclude micropip/_vendored/packaging/CHANGELOG.rst
+exclude micropip/_vendored/packaging/CONTRIBUTING.rst
+exclude micropip/_vendored/packaging/noxfile.py
+exclude micropip/_vendored/packaging/pyproject.toml
+
+include micropip/_vendored/packaging/LICENSE
+include micropip/_vendored/packaging/LICENSE.APACHE
+include micropip/_vendored/packaging/LICENSE.BSD
+
+include micropip/_vendored/packaging/README.rst

micropip/_compat/_compat_not_in_pyodide.py

@@ -14,6 +14,7 @@
 class CompatibilityNotInPyodide(CompatibilityLayer):
 
     # Vendored from packaging
+    # TODO: use packaging APIs here instead?
     _canonicalize_regex = re.compile(r"[-_.]+")
 
     class HttpStatusError(Exception):

micropip/_utils.py

@@ -4,14 +4,19 @@
 from pathlib import Path
 from sysconfig import get_config_var, get_platform
 
-from packaging.requirements import Requirement
-from packaging.tags import Tag
-from packaging.tags import sys_tags as sys_tags_orig
-from packaging.utils import BuildTag, InvalidWheelFilename, canonicalize_name
-from packaging.utils import parse_wheel_filename as parse_wheel_filename_orig
-from packaging.version import InvalidVersion, Version
-
 from ._compat import REPODATA_PACKAGES
+from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.tags import Tag
+from ._vendored.packaging.src.packaging.tags import sys_tags as sys_tags_orig
+from ._vendored.packaging.src.packaging.utils import (
+    BuildTag,
+    InvalidWheelFilename,
+    canonicalize_name,
+)
+from ._vendored.packaging.src.packaging.utils import (
+    parse_wheel_filename as parse_wheel_filename_orig,
+)
+from ._vendored.packaging.src.packaging.version import InvalidVersion, Version
 
 
 def get_dist_info(dist: Distribution) -> Path:

micropip/_vendored/packaging

@@ -7,13 +7,12 @@
 import warnings
 from typing import Any, Dict, List, Optional, Union, Literal, TypeAlias, TypedDict
 
-import packaging.utils
+import micropip._vendored.packaging.src.packaging.utils as packaging_utils
 
 
 ACCEPT_JSON_V1 = "application/vnd.pypi.simple.v1+json"
 
 
-
 class UnsupportedAPIVersion(Exception):
     """The major version of an API response is not supported."""
 
@@ -92,15 +91,15 @@ class ProjectDetails_1_0(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`691`)."""
 
     meta: _Meta_1_0
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_0]
 
 
 class ProjectDetails_1_1(TypedDict):
     """A :class:`~typing.TypedDict` for a project details response (:pep:`700`)."""
 
     meta: _Meta_1_1
-    name: packaging.utils.NormalizedName
+    name: packaging_utils.NormalizedName
     files: list[ProjectFileDetails_1_1]
     # PEP 700
     versions: List[str]
@@ -235,6 +234,6 @@ def from_project_details_html(html: str, name: str) -> ProjectDetails_1_0:
         files.append(details)
     return {
         "meta": {"api-version": "1.0"},
-        "name": packaging.utils.canonicalize_name(name),
+        "name": packaging_utils.canonicalize_name(name),
         "files": files,
     }

micropip/externals/mousebender/simple.py

@@ -4,9 +4,8 @@
 from copy import deepcopy
 from typing import Any
 
-from packaging.utils import canonicalize_name
-
 from ._utils import fix_package_dependencies
+from ._vendored.packaging.src.packaging.utils import canonicalize_name
 
 
 def freeze_lockfile(

micropip/freeze.py

@@ -4,9 +4,8 @@
 from pathlib import Path
 from typing import Any
 
-from packaging.markers import default_environment
-
 from ._compat import loadPackage, to_js
+from ._vendored.packaging.src.packaging.markers import default_environment
 from .constants import FAQ_URLS
 from .logging import setup_logging
 from .transaction import Transaction

micropip/install.py

@@ -7,8 +7,8 @@
 from collections.abc import Iterable
 from pathlib import Path
 
-from packaging.requirements import Requirement
-from packaging.utils import canonicalize_name
+from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.utils import canonicalize_name
 
 
 def safe_name(name):

micropip/metadata.py

@@ -3,7 +3,7 @@
 from dataclasses import astuple, dataclass
 from typing import Any
 
-from packaging.utils import canonicalize_name
+from ._vendored.packaging.src.packaging.utils import canonicalize_name
 
 __all__ = ["PackageDict"]
 

micropip/package.py

@@ -9,11 +9,10 @@
 from typing import Any
 from urllib.parse import urljoin, urlparse, urlunparse
 
-from packaging.utils import InvalidWheelFilename
-from packaging.version import InvalidVersion, Version
-
 from ._compat import HttpStatusError, fetch_string_and_headers
 from ._utils import is_package_compatible, parse_version
+from ._vendored.packaging.src.packaging.utils import InvalidWheelFilename
+from ._vendored.packaging.src.packaging.version import InvalidVersion, Version
 from .externals.mousebender.simple import from_project_details_html
 from .types import DistributionMetadata
 from .wheelinfo import WheelInfo

micropip/package_index.py

@@ -6,12 +6,11 @@
 from importlib.metadata import PackageNotFoundError
 from urllib.parse import urlparse
 
-from packaging.requirements import Requirement
-from packaging.utils import canonicalize_name
-
 from . import package_index
 from ._compat import REPODATA_PACKAGES
 from ._utils import best_compatible_tag_index, check_compatible
+from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.utils import canonicalize_name
 from .constants import FAQ_URLS
 from .package import PackageMetadata
 from .package_index import ProjectInfo

micropip/transaction.py

@@ -7,17 +7,16 @@
 from typing import Any, Literal
 from urllib.parse import ParseResult, urlparse
 
-from packaging.requirements import Requirement
-from packaging.tags import Tag
-from packaging.version import Version
-
 from ._compat import (
     fetch_bytes,
     get_dynlibs,
     loadDynlibsFromPackage,
     loadedPackages,
 )
 from ._utils import parse_wheel_filename
+from ._vendored.packaging.src.packaging.requirements import Requirement
+from ._vendored.packaging.src.packaging.tags import Tag
+from ._vendored.packaging.src.packaging.version import Version
 from .metadata import Metadata, safe_name, wheel_dist_info_dir
 from .types import DistributionMetadata
 

micropip/wheelinfo.py

@@ -13,7 +13,8 @@ classifiers = [
     "Operating System :: OS Independent",
 ]
 dynamic = ["version"]
-dependencies = ["packaging>=23.0"]
+dependencies = []
+
 [project.optional-dependencies]
 test = [
   "pytest-httpserver",
@@ -64,7 +65,12 @@ known-first-party = [
 ]
 
 [tool.mypy]
+exclude = ["micropip/_vendored/"]
 python_version = "3.12"
 show_error_codes = true
 warn_unreachable = true
 ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "micropip._vendored.*"
+warn_unreachable = false

pyproject.toml

@@ -10,10 +10,11 @@
 from typing import Any
 
 import pytest
-from packaging.utils import parse_wheel_filename
 from pytest_httpserver import HTTPServer
 from pytest_pyodide import spawn_web_server
 
+from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename
+
 
 def pytest_addoption(parser):
     parser.addoption(

tests/conftest.py

@@ -1,9 +1,9 @@
 import pytest
 from conftest import mock_fetch_cls
-from packaging.utils import parse_wheel_filename
 from pytest_pyodide import run_in_pyodide
 
 import micropip
+from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename
 
 
 def test_install_custom_url(selenium_standalone_micropip, wheel_catalog):

tests/test_install.py

@@ -1,6 +1,7 @@
 import pytest
 from conftest import SNOWBALL_WHEEL
-from packaging.tags import Tag
+
+from micropip._vendored.packaging.src.packaging.tags import Tag
 
 
 @pytest.mark.parametrize(
@@ -188,8 +189,7 @@ def _pypi_metadata(package, versions_to_tags):
 
 def test_last_version_from_pypi():
     pytest.importorskip("packaging")
-    from packaging.requirements import Requirement
-
+    from micropip._vendored.packaging.src.packaging.requirements import Requirement
     from micropip.transaction import find_wheel
 
     requirement = Requirement("dummy_module")
@@ -209,8 +209,7 @@ def test_find_wheel_invalid_version():
     it should be skipped instead of producing an error
     """
     pytest.importorskip("packaging")
-    from packaging.requirements import Requirement
-
+    from micropip._vendored.packaging.src.packaging.requirements import Requirement
     from micropip.transaction import find_wheel
 
     requirement = Requirement("dummy_module")
@@ -245,8 +244,7 @@ def test_find_wheel_invalid_version():
 @pytest.mark.parametrize(*_best_tag_test_cases)
 def test_best_tag_from_pypi(package, version, incompatible_tags, compatible_tags):
     pytest.importorskip("packaging")
-    from packaging.requirements import Requirement
-
+    from micropip._vendored.packaging.src.packaging.requirements import Requirement
     from micropip.transaction import find_wheel
 
     requirement = Requirement(package)
@@ -280,8 +278,7 @@ def test_last_version_and_best_tag_from_pypi(
     package, old_version, new_version, old_tags, new_tags
 ):
     pytest.importorskip("packaging")
-    from packaging.requirements import Requirement
-
+    from micropip._vendored.packaging.src.packaging.requirements import Requirement
     from micropip.transaction import find_wheel
 
     requirement = Requirement(package)

tests/test_transaction.py

@@ -1,7 +1,7 @@
 # isort: skip_file
 
 from pytest_pyodide import run_in_pyodide
-from packaging.utils import parse_wheel_filename
+from micropip._vendored.packaging.src.packaging.utils import parse_wheel_filename
 
 TEST_PACKAGE_NAME = "test-wheel-uninstall"