Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor installer extras handling, fix small bug, drop support for very old lock files #9345

Merged
merged 2 commits into from
Jun 2, 2024
Merged

refactor installer extras handling, fix small bug, drop support for very old lock files #9345

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bd3500d
installer: simplify logic for calculating operations regarding extras…
radoering Apr 28, 2024
1615268
locker/installer: drop support for reading very old lock files (prior…
radoering Apr 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 16 additions & 117 deletions src/poetry/installation/installer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,12 @@
from packaging.utils import canonicalize_name

from poetry.installation.executor import Executor
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update
from poetry.repositories import Repository
from poetry.repositories import RepositoryPool
from poetry.repositories.installed_repository import InstalledRepository
from poetry.repositories.lockfile_repository import LockfileRepository
from poetry.utils.extras import get_extra_package_names


if TYPE_CHECKING:
Expand Down Expand Up @@ -239,15 +237,17 @@ def _do_install(self) -> int:
source_root=self._env.path.joinpath("src")
):
ops = solver.solve(use_latest=self._whitelist).calculate_operations()

lockfile_repo = LockfileRepository()
self._populate_lockfile_repo(lockfile_repo, ops)

else:
self._io.write_line("<info>Installing dependencies from lock file</>")

locked_repository = self._locker.locked_repository()

if not self._locker.is_fresh():
raise ValueError(
"pyproject.toml changed significantly since poetry.lock was last generated. "
"Run `poetry lock [--no-update]` to fix the lock file."
"pyproject.toml changed significantly since poetry.lock was last"
" generated. Run `poetry lock [--no-update]` to fix the lock file."
)

locker_extras = {
Expand All @@ -258,30 +258,25 @@ def _do_install(self) -> int:
if extra not in locker_extras:
raise ValueError(f"Extra [{extra}] is not specified.")

# If we are installing from lock
# Filter the operations by comparing it with what is
# currently installed
ops = self._get_operations_from_lock(locked_repository)

lockfile_repo = LockfileRepository()
uninstalls = self._populate_lockfile_repo(lockfile_repo, ops)
locked_repository = self._locker.locked_repository()
lockfile_repo = locked_repository

if not self.executor.enabled:
# If we are only in lock mode, no need to go any further
self._write_lock_file(lockfile_repo)
return 0

if self._groups is not None:
root = self._package.with_dependency_groups(list(self._groups), only=True)
else:
root = self._package.without_optional_dependency_groups()

if self._io.is_verbose():
self._io.write_line("")
self._io.write_line(
"<info>Finding the necessary packages for the current system</>"
)

if self._groups is not None:
root = self._package.with_dependency_groups(list(self._groups), only=True)
else:
root = self._package.without_optional_dependency_groups()

# We resolve again by only using the lock file
packages = lockfile_repo.packages + locked_repository.packages
pool = RepositoryPool.from_packages(packages, self._config)
Expand All @@ -299,36 +294,12 @@ def _do_install(self) -> int:

with solver.use_environment(self._env):
ops = solver.solve(use_latest=self._whitelist).calculate_operations(
with_uninstalls=self._requires_synchronization,
with_uninstalls=self._requires_synchronization or self._update,
synchronize=self._requires_synchronization,
skip_directory=self._skip_directory,
extras=set(self._extras),
)

if not self._requires_synchronization:
# If no packages synchronisation has been requested we need
# to calculate the uninstall operations
from poetry.puzzle.transaction import Transaction

transaction = Transaction(
locked_repository.packages,
[(package, 0) for package in lockfile_repo.packages],
installed_packages=self._installed_repository.packages,
root_package=root,
)

ops = [
op
for op in transaction.calculate_operations(with_uninstalls=True)
if op.job_type == "uninstall"
] + ops
else:
ops = uninstalls + ops

# We need to filter operations so that packages
# not compatible with the current system,
# or optional and not requested, are dropped
self._filter_operations(ops, lockfile_repo)

# Validate the dependencies
for op in ops:
dep = op.package.to_dependency()
Expand Down Expand Up @@ -358,86 +329,14 @@ def _execute(self, operations: list[Operation]) -> int:

def _populate_lockfile_repo(
self, repo: LockfileRepository, ops: Iterable[Operation]
) -> list[Uninstall]:
uninstalls = []
) -> None:
for op in ops:
if isinstance(op, Uninstall):
uninstalls.append(op)
continue

package = op.target_package if isinstance(op, Update) else op.package
if not repo.has_package(package):
repo.add_package(package)

return uninstalls

def _get_operations_from_lock(
self, locked_repository: Repository
) -> list[Operation]:
installed_repo = self._installed_repository
ops: list[Operation] = []

extra_packages = self._get_extra_packages(locked_repository)
for locked in locked_repository.packages:
is_installed = False
for installed in installed_repo.packages:
if locked.name == installed.name:
is_installed = True
if locked.optional and locked.name not in extra_packages:
# Installed but optional and not requested in extras
ops.append(Uninstall(locked))
elif locked.version != installed.version:
ops.append(Update(installed, locked))

# If it's optional and not in required extras
# we do not install
if locked.optional and locked.name not in extra_packages:
continue

op = Install(locked)
if is_installed:
op.skip("Already installed")

ops.append(op)

return ops

def _filter_operations(self, ops: Iterable[Operation], repo: Repository) -> None:
extra_packages = self._get_extra_packages(repo)
for op in ops:
package = op.target_package if isinstance(op, Update) else op.package

if op.job_type == "uninstall":
continue

if not self._env.is_valid_for_marker(package.marker):
op.skip("Not needed for the current environment")
continue

# If a package is optional and not requested
# in any extra we skip it
if package.optional and package.name not in extra_packages:
op.skip("Not required")

def _get_extra_packages(self, repo: Repository) -> set[NormalizedName]:
"""
Returns all package names required by extras.

Maybe we just let the solver handle it?
"""
extras: dict[NormalizedName, list[NormalizedName]]
if self._update:
extras = {k: [d.name for d in v] for k, v in self._package.extras.items()}
else:
raw_extras = self._locker.lock_data.get("extras", {})
extras = {
canonicalize_name(extra): [
canonicalize_name(dependency) for dependency in dependencies
]
for extra, dependencies in raw_extras.items()
}

return get_extra_package_names(repo.packages, extras, self._extras)

def _get_installed(self) -> InstalledRepository:
return InstalledRepository.load(self._env)
24 changes: 6 additions & 18 deletions src/poetry/packages/locker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
from poetry.core.constraints.version import parse_constraint
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.package import Package
from poetry.core.version.markers import parse_marker
from poetry.core.version.requirements import InvalidRequirement
from tomlkit import array
from tomlkit import comment
Expand Down Expand Up @@ -205,22 +204,6 @@ def locked_repository(self) -> LockfileRepository:

package.extras = package_extras

if "marker" in info:
package.marker = parse_marker(info["marker"])
else:
# Compatibility for old locks
if "requirements" in info:
dep = Dependency("foo", "0.0.0")
for name, value in info["requirements"].items():
if name == "python":
dep.python_versions = value
elif name == "platform":
dep.platform = value

split_dep = dep.to_pep_508(False).split(";")
if len(split_dep) > 1:
package.marker = parse_marker(split_dep[1].strip())

for dep_name, constraint in info.get("dependencies", {}).items():
root_dir = self.lock.parent
if package.source_type == "directory":
Expand Down Expand Up @@ -348,7 +331,12 @@ def _get_lock_data(self) -> dict[str, Any]:
)

metadata = lock_data["metadata"]
lock_version = Version.parse(metadata.get("lock-version", "1.0"))
if "lock-version" not in metadata:
raise RuntimeError(
"The lock file is not compatible with the current version of Poetry.\n"
"Regenerate the lock file with the `poetry lock` command."
)
lock_version = Version.parse(metadata["lock-version"])
current_version = Version.parse(self._VERSION)
accepted_versions = parse_constraint(self._READ_VERSION_RANGE)
lock_version_allowed = accepted_versions.allows(lock_version)
Expand Down
32 changes: 28 additions & 4 deletions src/poetry/puzzle/transaction.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,11 @@

from typing import TYPE_CHECKING

from poetry.utils.extras import get_extra_package_names


if TYPE_CHECKING:
from packaging.utils import NormalizedName
from poetry.core.packages.package import Package

from poetry.installation.operations.operation import Operation
Expand Down Expand Up @@ -32,23 +35,42 @@ def calculate_operations(
synchronize: bool = False,
*,
skip_directory: bool = False,
extras: set[NormalizedName] | None = None,
) -> list[Operation]:
from poetry.installation.operations import Install
from poetry.installation.operations import Uninstall
from poetry.installation.operations import Update

operations: list[Operation] = []

extra_packages: set[NormalizedName] = set()
if extras is not None:
assert self._root_package is not None
extra_packages = get_extra_package_names(
[package for package, _ in self._result_packages],
{k: [d.name for d in v] for k, v in self._root_package.extras.items()},
extras,
)

uninstalls: set[NormalizedName] = set()
for result_package, priority in self._result_packages:
installed = False
is_unsolicited_extra = extras is not None and (
result_package.optional and result_package.name not in extra_packages
)

for installed_package in self._installed_packages:
if result_package.name == installed_package.name:
installed = True

# Extras that were not requested are always uninstalled.
if is_unsolicited_extra:
uninstalls.add(installed_package.name)
operations.append(Uninstall(installed_package))

# We have to perform an update if the version or another
# attribute of the package has changed (source type, url, ref, ...).
if result_package.version != installed_package.version or (
elif result_package.version != installed_package.version or (
(
# This has to be done because installed packages cannot
# have type "legacy". If a package with type "legacy"
Expand Down Expand Up @@ -78,10 +100,12 @@ def calculate_operations(
installed
or (skip_directory and result_package.source_type == "directory")
):
operations.append(Install(result_package, priority=priority))
op = Install(result_package, priority=priority)
if is_unsolicited_extra:
op.skip("Not required")
operations.append(op)

if with_uninstalls:
uninstalls: set[str] = set()
for current_package in self._current_packages:
found = any(
current_package.name == result_package.name
Expand All @@ -92,7 +116,7 @@ def calculate_operations(
for installed_package in self._installed_packages:
if installed_package.name == current_package.name:
uninstalls.add(installed_package.name)
operations.append(Uninstall(current_package))
operations.append(Uninstall(installed_package))

if synchronize:
result_package_names = {
Expand Down
5 changes: 2 additions & 3 deletions tests/console/commands/self/test_remove_plugins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,13 @@ def install_plugin(installed: Repository) -> None:
"optional": False,
"platform": "*",
"python-versions": "*",
"checksum": [],
"files": [],
},
],
"metadata": {
"lock-version": "2.0",
"python-versions": "^3.6",
"platform": "*",
"content-hash": "123456789",
"files": {"poetry-plugin": []},
},
}
system_pyproject_file.parent.joinpath("poetry.lock").write_text(
Expand Down
Loading
Loading