diff --git a/.gitignore b/.gitignore index fcbd92e97a3..0004c5bcc85 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ MANIFEST.in /releases/* pip-wheel-metadata /poetry.toml + +poetry/core/* diff --git a/docs/configuration.md b/docs/configuration.md index 327a8d28710..c93322310a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -123,6 +123,21 @@ Defaults to one of the following directories: Use parallel execution when using the new (`>=1.1.0`) installer. Defaults to `true`. +### `installer.max-workers` + +**Type**: int + +Set the maximum number of workers while using the parallel installer. Defaults to `number_of_cores + 4`. +The `number_of_cores` is determined by `os.cpu_count()`. +If this raises a `NotImplentedError` exception `number_of_cores` is assumed to be 1. + +If this configuration parameter is set to a value greater than `number_of_cores + 4`, +the number of maximum workers is still limited at `number_of_cores + 4`. + +{{% note %}} +This configuration will be ignored when `installer.parallel` is set to false. +{{% /note %}} + ### `virtualenvs.create` **Type**: boolean diff --git a/poetry.lock b/poetry.lock index e1987a36d8c..36b6f769f8b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -211,6 +211,14 @@ python-versions = ">=3.6" docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +[[package]] +name = "flatdict" +version = "4.0.1" +description = "Python module for interacting with nested dicts as a single level dict with delimited keys." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "html5lib" version = "1.1" @@ -739,7 +747,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "9d2e32899df46f2c63018e9a3f5e95dbbeb1ec41291c31289cff40f6f2d935a4" +content-hash = "d427df125a868ada92bbb6d3a8cc90def6034ad684c1546afb519729048ab150" [metadata.files] atomicwrites = [ @@ -929,6 +937,9 @@ filelock = [ {file = "filelock-3.4.0-py3-none-any.whl", hash = "sha256:2e139a228bcf56dd8b2274a65174d005c4a6b68540ee0bdbb92c76f43f29f7e8"}, {file = "filelock-3.4.0.tar.gz", hash = "sha256:93d512b32a23baf4cac44ffd72ccf70732aeff7b8050fcaf6d3ec406d954baf4"}, ] +flatdict = [ + {file = "flatdict-4.0.1.tar.gz", hash = "sha256:cd32f08fd31ed21eb09ebc76f06b6bd12046a24f77beb1fd0281917e47f26742"}, +] html5lib = [ {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, diff --git a/pyproject.toml b/pyproject.toml index 4ff1a3f4420..0dc64b4877c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ deepdiff = "^5.0" httpretty = "^1.0" typing-extensions = { version = "^4.0.0", python = "<3.8" } zipp = { version = "^3.4", python = "<3.8" } +flatdict = "^4.0.1" [tool.poetry.scripts] poetry = "poetry.console.application:main" diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 3ba4f2c4c0a..720e479f48f 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -25,6 +25,10 @@ def boolean_normalizer(val: str) -> bool: return val in ["true", "1"] +def int_normalizer(val: str) -> int: + return int(val) + + class Config: default_config: Dict[str, Any] = { @@ -36,7 +40,7 @@ class Config: "options": {"always-copy": False, "system-site-packages": False}, }, "experimental": {"new-installer": True}, - "installer": {"parallel": True}, + "installer": {"parallel": True, "max-workers": None}, } def __init__( @@ -129,7 +133,8 @@ def process(self, value: Any) -> Any: return re.sub(r"{(.+?)}", lambda m: self.get(m.group(1)), value) - def _get_normalizer(self, name: str) -> Callable: + @staticmethod + def _get_normalizer(name: str) -> Callable: if name in { "virtualenvs.create", "virtualenvs.in-project", @@ -143,4 +148,7 @@ def _get_normalizer(self, name: str) -> Callable: if name == "virtualenvs.path": return lambda val: str(Path(val)) + if name == "installer.max-workers": + return int_normalizer + return lambda val: val diff --git a/src/poetry/console/commands/config.py b/src/poetry/console/commands/config.py index a6dbed969ff..e30e0ef3027 100644 --- a/src/poetry/console/commands/config.py +++ b/src/poetry/console/commands/config.py @@ -52,6 +52,7 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: from poetry.config.config import boolean_normalizer from poetry.config.config import boolean_validator + from poetry.config.config import int_normalizer from poetry.locations import CACHE_DIR unique_config_values = { @@ -87,6 +88,11 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: boolean_normalizer, True, ), + "installer.max-workers": ( + lambda val: int(val) > 0, + int_normalizer, + None, + ), } return unique_config_values diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 7aba1c678be..df34887b487 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -12,6 +12,7 @@ from typing import Any from typing import Dict from typing import List +from typing import Optional from typing import Union from cleo.io.null_io import NullIO @@ -66,14 +67,9 @@ def __init__( parallel = config.get("installer.parallel", True) if parallel: - # This should be directly handled by ThreadPoolExecutor - # however, on some systems the number of CPUs cannot be determined - # (it raises a NotImplementedError), so, in this case, we assume - # that the system only has one CPU. - try: - self._max_workers = os.cpu_count() + 4 - except NotImplementedError: - self._max_workers = 5 + self._max_workers = self._get_max_workers( + desired_max_workers=config.get("installer.max-workers") + ) else: self._max_workers = 1 @@ -190,6 +186,21 @@ def execute(self, operations: List["OperationTypes"]) -> int: return 1 if self._shutdown else 0 + @staticmethod + def _get_max_workers(desired_max_workers: Optional[int] = None): + # This should be directly handled by ThreadPoolExecutor + # however, on some systems the number of CPUs cannot be determined + # (it raises a NotImplementedError), so, in this case, we assume + # that the system only has one CPU. + try: + default_max_workers = os.cpu_count() + 4 + except NotImplementedError: + default_max_workers = 5 + + if desired_max_workers is None: + return default_max_workers + return min(default_max_workers, desired_max_workers) + def _write(self, operation: "OperationTypes", line: str) -> None: if not self.supports_fancy_output() or not self._should_write_operation( operation diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 6f05056fb19..0f8c237c27d 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -2,31 +2,29 @@ import re from typing import TYPE_CHECKING -from typing import Any -from typing import Dict +from typing import Callable from typing import Iterator -from typing import Optional from typing import Tuple import pytest +from flatdict import FlatDict + from poetry.config.config import Config +from poetry.config.config import boolean_normalizer +from poetry.config.config import int_normalizer if TYPE_CHECKING: from pathlib import Path -def get_boolean_options(config: Optional[Dict[str, Any]] = None) -> str: - if config is None: - config = Config.default_config +def get_options_based_on_normalizer(normalizer: Callable) -> str: + flattened_config = FlatDict(Config.default_config, delimiter=".") - for k, v in config.items(): - if isinstance(v, bool) or v is None: + for k in flattened_config: + if Config._get_normalizer(k) == normalizer: yield k - if isinstance(v, dict): - for suboption in get_boolean_options(v): - yield f"{k}.{suboption}" @pytest.mark.parametrize( @@ -43,10 +41,14 @@ def test_config_get_processes_depended_on_values( def generate_environment_variable_tests() -> Iterator[Tuple[str, str, str, bool]]: - for env_value, value in [("true", True), ("false", False)]: - for name in get_boolean_options(): - env_var = "POETRY_{}".format(re.sub("[.-]+", "_", name).upper()) - yield name, env_var, env_value, value + for normalizer, values in [ + (boolean_normalizer, [("true", True), ("false", False)]), + (int_normalizer, [("4", 4), ("2", 2)]), + ]: + for env_value, value in values: + for name in get_options_based_on_normalizer(normalizer=normalizer): + env_var = "POETRY_" + re.sub("[.-]+", "_", name).upper() + yield name, env_var, env_value, value @pytest.mark.parametrize( diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 8d9620b1b28..d4a2dfec857 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -46,6 +46,7 @@ def test_list_displays_default_value_if_not_set( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = true virtualenvs.in-project = null @@ -70,6 +71,7 @@ def test_list_displays_set_get_setting( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null @@ -118,6 +120,7 @@ def test_list_displays_set_get_local_setting( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 5da4ef2d492..8b9fdcf03ac 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -494,3 +494,35 @@ def test_executor_should_use_cached_link_and_hash( Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"), ) assert archive == link_cached + + +@pytest.mark.parametrize( + ("max_workers", "cpu_count", "side_effect", "expected_workers"), + [ + (None, 3, None, 7), + (3, 4, None, 3), + (8, 3, None, 7), + (None, 8, NotImplementedError(), 5), + (2, 8, NotImplementedError(), 2), + (8, 8, NotImplementedError(), 5), + ], +) +def test_executor_should_be_initialized_with_correct_workers( + tmp_venv, + pool, + config, + io, + mocker, + max_workers, + cpu_count, + side_effect, + expected_workers, +): + config = Config() + config.merge({"installer": {"max-workers": max_workers}}) + + mocker.patch("os.cpu_count", return_value=cpu_count, side_effect=side_effect) + + executor = Executor(tmp_venv, pool, config, io) + + assert executor._max_workers == expected_workers