From cffef31906456909d304e3af96fcec4656e370d6 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 27 Dec 2020 17:17:47 +0100 Subject: [PATCH 01/11] Add maximum of workers to config --- docs/docs/configuration.md | 7 +++++++ poetry/console/commands/config.py | 6 ++++++ poetry/installation/executor.py | 17 ++++++++--------- poetry/utils/helpers.py | 13 +++++++++++++ 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/docs/docs/configuration.md b/docs/docs/configuration.md index 72310886ca7..caf24416a1f 100644 --- a/docs/docs/configuration.md +++ b/docs/docs/configuration.md @@ -112,6 +112,13 @@ Defaults to `true`. This configuration will be ignored, and parallel execution disabled when running Python 2.7 under Windows. +### `installer.max-workers`: int + +Set the maximum number of workers while using the parallel installer. Defaults to `number_of_cores + 4`. + +!!!note: + This configuration will be ignored when `installer.parallel` is set to false. + ### `virtualenvs.create`: boolean Create a new virtual environment if one doesn't already exist. diff --git a/poetry/console/commands/config.py b/poetry/console/commands/config.py index 310524d4ee2..9a71e3552b4 100644 --- a/poetry/console/commands/config.py +++ b/poetry/console/commands/config.py @@ -46,6 +46,7 @@ def unique_config_values(self): from poetry.config.config import boolean_normalizer from poetry.config.config import boolean_validator from poetry.locations import CACHE_DIR + from poetry.utils.helpers import get_default_max_workers unique_config_values = { "cache-dir": ( @@ -71,6 +72,11 @@ def unique_config_values(self): False, ), "installer.parallel": (boolean_validator, boolean_normalizer, True,), + "installer.max-workers": ( + lambda val: int(val) > 0, + lambda val: int(val), + get_default_max_workers(), + ), } return unique_config_values diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 5523ed2813f..188de6371a3 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -16,6 +16,7 @@ from poetry.io.null_io import NullIO from poetry.utils._compat import decode from poetry.utils.env import EnvCommandError +from poetry.utils.helpers import get_default_max_workers from poetry.utils.helpers import safe_rmtree from .authenticator import Authenticator @@ -28,7 +29,7 @@ class Executor(object): - def __init__(self, env, pool, config, io, parallel=None): + def __init__(self, env, pool, config, io, parallel=None, max_workers=None): self._env = env self._io = io self._dry_run = False @@ -42,14 +43,12 @@ def __init__(self, env, pool, config, io, parallel=None): parallel = config.get("installer.parallel", True) if parallel: - # This should be directly handled by ThreadPoolExecutor - # however, on some systems the number of CPUs cannot be determined - # (it raises a NotImplementedError), so, in this case, we assume - # that the system only has one CPU. - try: - self._max_workers = os.cpu_count() + 4 - except NotImplementedError: - self._max_workers = 5 + self._max_workers = get_default_max_workers() + + if max_workers is None: + max_workers = config.get("installer.max-workers", self._max_workers) + + self._max_workers = min(max_workers, self._max_workers) else: self._max_workers = 1 diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index 909ec3c1438..55ed6bf3567 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -133,3 +133,16 @@ def is_dir_writable(path, create=False): # type: (Path, bool) -> bool return False else: return True + + +def get_default_max_workers(): + # This should be directly handled by ThreadPoolExecutor + # however, on some systems the number of CPUs cannot be determined + # (it raises a NotImplementedError), so, in this case, we assume + # that the system only has one CPU. + try: + default_max_workers = os.cpu_count() + 4 + except NotImplementedError: + default_max_workers = 5 + + return default_max_workers From 7bc6a22b5148a47ba1934b92177b07f2a003ce6e Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 27 Dec 2020 17:39:05 +0100 Subject: [PATCH 02/11] Trigger build --- poetry/installation/executor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 188de6371a3..2368dc33fdb 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -43,6 +43,7 @@ def __init__(self, env, pool, config, io, parallel=None, max_workers=None): parallel = config.get("installer.parallel", True) if parallel: + # Trigger build self._max_workers = get_default_max_workers() if max_workers is None: From a48fd789dfd3c3dff273eb170076f12460ab944f Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 27 Dec 2020 17:45:31 +0100 Subject: [PATCH 03/11] Revert Trigger build --- poetry/installation/executor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 2368dc33fdb..188de6371a3 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -43,7 +43,6 @@ def __init__(self, env, pool, config, io, parallel=None, max_workers=None): parallel = config.get("installer.parallel", True) if parallel: - # Trigger build self._max_workers = get_default_max_workers() if max_workers is None: From 5b16a4108de0ba9dece6536a216395f08f949fdd Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Mon, 28 Dec 2020 11:42:54 +0100 Subject: [PATCH 04/11] Add test for config --- tests/config/test_config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/config/test_config.py b/tests/config/test_config.py index f3b13f23003..7727795ecfe 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -19,11 +19,15 @@ def test_config_get_processes_depended_on_values(config, config_cache_dir): [ ("installer.parallel", "true", True), ("installer.parallel", "false", False), + ("installer.max-workers", "4", "4"), + ("installer.max-workers", "2", "2"), ("virtualenvs.create", "true", True), ("virtualenvs.create", "false", False), ], ) def test_config_get_from_environment_variable(config, environ, name, env_value, value): - env_var = "POETRY_{}".format("_".join(k.upper() for k in name.split("."))) + env_var = "POETRY_{}".format( + "_".join(k.upper().replace("-", "_") for k in name.split(".")) + ) os.environ[env_var] = env_value assert config.get(name) is value From 8c10e9c592dea29eebeea20b9cf3552c3c36e787 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 14 Nov 2021 14:11:15 +0100 Subject: [PATCH 05/11] Address max workers logic --- poetry/console/commands/config.py | 4 ++-- poetry/installation/executor.py | 12 ++++-------- poetry/utils/helpers.py | 6 ++++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/poetry/console/commands/config.py b/poetry/console/commands/config.py index 5558b2924f1..799fd1a218e 100644 --- a/poetry/console/commands/config.py +++ b/poetry/console/commands/config.py @@ -53,7 +53,7 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: from poetry.config.config import boolean_normalizer from poetry.config.config import boolean_validator from poetry.locations import CACHE_DIR - from poetry.utils.helpers import get_default_max_workers + from poetry.utils.helpers import get_max_workers unique_config_values = { "cache-dir": ( @@ -91,7 +91,7 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: "installer.max-workers": ( lambda val: int(val) > 0, lambda val: int(val), - get_default_max_workers(), + get_max_workers(), ), } diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index add2bff78e4..dbdfc5050d4 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -13,6 +13,7 @@ from typing import Dict from typing import List from typing import Union +from typing import Optional from cleo.io.null_io import NullIO @@ -23,7 +24,7 @@ from poetry.core.pyproject.toml import PyProjectTOML from poetry.utils._compat import decode from poetry.utils.env import EnvCommandError -from poetry.utils.helpers import get_default_max_workers +from poetry.utils.helpers import get_max_workers from poetry.utils.helpers import safe_rmtree from poetry.utils.pip import pip_editable_install @@ -55,7 +56,7 @@ def __init__( config: "Config", io: "IO", parallel: bool = None, - max_workers: int = None, + max_workers: Optional[int] = None, ) -> None: self._env = env self._io = io @@ -70,12 +71,7 @@ def __init__( parallel = config.get("installer.parallel", True) if parallel: - self._max_workers = get_default_max_workers() - - if max_workers is None: - max_workers = config.get("installer.max-workers", self._max_workers) - - self._max_workers = min(max_workers, self._max_workers) + self._max_workers = config.get("installer.max-workers", get_max_workers(desired_max_workers=max_workers)) else: self._max_workers = 1 diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index 3c82450b2c2..ba2e7c6f5b6 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -143,7 +143,7 @@ def is_dir_writable(path: Path, create: bool = False) -> bool: return True -def get_default_max_workers(): +def get_max_workers(desired_max_workers: Optional[int] = None): # This should be directly handled by ThreadPoolExecutor # however, on some systems the number of CPUs cannot be determined # (it raises a NotImplementedError), so, in this case, we assume @@ -153,4 +153,6 @@ def get_default_max_workers(): except NotImplementedError: default_max_workers = 5 - return default_max_workers + if desired_max_workers is None: + return default_max_workers + return min(default_max_workers, desired_max_workers) From ddb2a6587113878d18091b80854cb792cf32b209 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 14 Nov 2021 13:12:07 +0000 Subject: [PATCH 06/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- poetry/installation/executor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index dbdfc5050d4..7c05bfd011b 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -12,8 +12,8 @@ from typing import Any from typing import Dict from typing import List -from typing import Union from typing import Optional +from typing import Union from cleo.io.null_io import NullIO @@ -71,7 +71,10 @@ def __init__( parallel = config.get("installer.parallel", True) if parallel: - self._max_workers = config.get("installer.max-workers", get_max_workers(desired_max_workers=max_workers)) + self._max_workers = config.get( + "installer.max-workers", + get_max_workers(desired_max_workers=max_workers), + ) else: self._max_workers = 1 From 4aa81176aa4ee02d86fbf8e9073c81b7a123e55b Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sat, 20 Nov 2021 14:15:48 +0100 Subject: [PATCH 07/11] Update maximum worker logic --- .gitignore | 2 ++ docs/configuration.md | 5 +++++ poetry/config/config.py | 9 +++++++- poetry/console/commands/config.py | 6 +++--- poetry/installation/executor.py | 22 +++++++++++++++----- poetry/utils/helpers.py | 15 -------------- tests/config/test_config.py | 4 ++-- tests/installation/test_executor.py | 32 +++++++++++++++++++++++++++++ 8 files changed, 69 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index fcbd92e97a3..0004c5bcc85 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ MANIFEST.in /releases/* pip-wheel-metadata /poetry.toml + +poetry/core/* diff --git a/docs/configuration.md b/docs/configuration.md index 19b988c969a..c93322310a1 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -128,6 +128,11 @@ Defaults to `true`. **Type**: int Set the maximum number of workers while using the parallel installer. Defaults to `number_of_cores + 4`. +The `number_of_cores` is determined by `os.cpu_count()`. +If this raises a `NotImplentedError` exception `number_of_cores` is assumed to be 1. + +If this configuration parameter is set to a value greater than `number_of_cores + 4`, +the number of maximum workers is still limited at `number_of_cores + 4`. {{% note %}} This configuration will be ignored when `installer.parallel` is set to false. diff --git a/poetry/config/config.py b/poetry/config/config.py index b1b3b3e61bb..3f7a93872de 100644 --- a/poetry/config/config.py +++ b/poetry/config/config.py @@ -25,6 +25,10 @@ def boolean_normalizer(val: str) -> bool: return val in ["true", "1"] +def int_normalizer(val: str) -> int: + return int(val) + + class Config: default_config = { @@ -36,7 +40,7 @@ class Config: "options": {"always-copy": False, "system-site-packages": False}, }, "experimental": {"new-installer": True}, - "installer": {"parallel": True}, + "installer": {"parallel": True, "max-workers": None}, } def __init__( @@ -146,4 +150,7 @@ def _get_normalizer(self, name: str) -> Callable: if name == "virtualenvs.path": return lambda val: str(Path(val)) + if name == "installer.max-workers": + return int_normalizer + return lambda val: val diff --git a/poetry/console/commands/config.py b/poetry/console/commands/config.py index 799fd1a218e..95672b4ec16 100644 --- a/poetry/console/commands/config.py +++ b/poetry/console/commands/config.py @@ -52,8 +52,8 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: from poetry.config.config import boolean_normalizer from poetry.config.config import boolean_validator + from poetry.config.config import int_normalizer from poetry.locations import CACHE_DIR - from poetry.utils.helpers import get_max_workers unique_config_values = { "cache-dir": ( @@ -90,8 +90,8 @@ def unique_config_values(self) -> Dict[str, Tuple[Any, Any, Any]]: ), "installer.max-workers": ( lambda val: int(val) > 0, - lambda val: int(val), - get_max_workers(), + int_normalizer, + None, ), } diff --git a/poetry/installation/executor.py b/poetry/installation/executor.py index 7c05bfd011b..18dc7ee70c9 100644 --- a/poetry/installation/executor.py +++ b/poetry/installation/executor.py @@ -24,7 +24,6 @@ from poetry.core.pyproject.toml import PyProjectTOML from poetry.utils._compat import decode from poetry.utils.env import EnvCommandError -from poetry.utils.helpers import get_max_workers from poetry.utils.helpers import safe_rmtree from poetry.utils.pip import pip_editable_install @@ -56,7 +55,6 @@ def __init__( config: "Config", io: "IO", parallel: bool = None, - max_workers: Optional[int] = None, ) -> None: self._env = env self._io = io @@ -71,9 +69,8 @@ def __init__( parallel = config.get("installer.parallel", True) if parallel: - self._max_workers = config.get( - "installer.max-workers", - get_max_workers(desired_max_workers=max_workers), + self._max_workers = self._get_max_workers( + desired_max_workers=config.get("installer.max-workers") ) else: self._max_workers = 1 @@ -191,6 +188,21 @@ def execute(self, operations: List["OperationTypes"]) -> int: return 1 if self._shutdown else 0 + @staticmethod + def _get_max_workers(desired_max_workers: Optional[int] = None): + # This should be directly handled by ThreadPoolExecutor + # however, on some systems the number of CPUs cannot be determined + # (it raises a NotImplementedError), so, in this case, we assume + # that the system only has one CPU. + try: + default_max_workers = os.cpu_count() + 4 + except NotImplementedError: + default_max_workers = 5 + + if desired_max_workers is None: + return default_max_workers + return min(default_max_workers, desired_max_workers) + def _write(self, operation: "OperationTypes", line: str) -> None: if not self.supports_fancy_output() or not self._should_write_operation( operation diff --git a/poetry/utils/helpers.py b/poetry/utils/helpers.py index ba2e7c6f5b6..a19874a0ce2 100644 --- a/poetry/utils/helpers.py +++ b/poetry/utils/helpers.py @@ -141,18 +141,3 @@ def is_dir_writable(path: Path, create: bool = False) -> bool: return False else: return True - - -def get_max_workers(desired_max_workers: Optional[int] = None): - # This should be directly handled by ThreadPoolExecutor - # however, on some systems the number of CPUs cannot be determined - # (it raises a NotImplementedError), so, in this case, we assume - # that the system only has one CPU. - try: - default_max_workers = os.cpu_count() + 4 - except NotImplementedError: - default_max_workers = 5 - - if desired_max_workers is None: - return default_max_workers - return min(default_max_workers, desired_max_workers) diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 7727795ecfe..9f8f5237e52 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -19,8 +19,8 @@ def test_config_get_processes_depended_on_values(config, config_cache_dir): [ ("installer.parallel", "true", True), ("installer.parallel", "false", False), - ("installer.max-workers", "4", "4"), - ("installer.max-workers", "2", "2"), + ("installer.max-workers", "4", 4), + ("installer.max-workers", "2", 2), ("virtualenvs.create", "true", True), ("virtualenvs.create", "false", False), ], diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 5da4ef2d492..8b9fdcf03ac 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -494,3 +494,35 @@ def test_executor_should_use_cached_link_and_hash( Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"), ) assert archive == link_cached + + +@pytest.mark.parametrize( + ("max_workers", "cpu_count", "side_effect", "expected_workers"), + [ + (None, 3, None, 7), + (3, 4, None, 3), + (8, 3, None, 7), + (None, 8, NotImplementedError(), 5), + (2, 8, NotImplementedError(), 2), + (8, 8, NotImplementedError(), 5), + ], +) +def test_executor_should_be_initialized_with_correct_workers( + tmp_venv, + pool, + config, + io, + mocker, + max_workers, + cpu_count, + side_effect, + expected_workers, +): + config = Config() + config.merge({"installer": {"max-workers": max_workers}}) + + mocker.patch("os.cpu_count", return_value=cpu_count, side_effect=side_effect) + + executor = Executor(tmp_venv, pool, config, io) + + assert executor._max_workers == expected_workers From 474e879db400181015ceebd065876de96502343a Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sat, 20 Nov 2021 14:50:10 +0100 Subject: [PATCH 08/11] Update test_config.py to include int tests --- tests/config/test_config.py | 41 +++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 6f05056fb19..a1c662c305f 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from typing import Any +from typing import Callable from typing import Dict from typing import Iterator from typing import Optional @@ -11,22 +12,36 @@ import pytest from poetry.config.config import Config +from poetry.config.config import boolean_normalizer +from poetry.config.config import int_normalizer if TYPE_CHECKING: from pathlib import Path -def get_boolean_options(config: Optional[Dict[str, Any]] = None) -> str: +def flatten(config: Optional[Dict[str, Any]], parent_key: str = "", sep="."): + items = [] + for k, v in config.items(): + new_key = parent_key + sep + k if parent_key else k + if isinstance(v, dict): + items.extend(flatten(v, new_key, sep=sep).items()) + else: + items.append((new_key, v)) + return dict(items) + + +def get_options_based_on_normalizer( + normalizer: Callable, config: Optional[Config] = None +) -> str: if config is None: - config = Config.default_config + config = Config() - for k, v in config.items(): - if isinstance(v, bool) or v is None: + flattened_config = flatten(config.default_config) + + for k in flattened_config: + if config._get_normalizer(k) == normalizer: yield k - if isinstance(v, dict): - for suboption in get_boolean_options(v): - yield f"{k}.{suboption}" @pytest.mark.parametrize( @@ -43,10 +58,14 @@ def test_config_get_processes_depended_on_values( def generate_environment_variable_tests() -> Iterator[Tuple[str, str, str, bool]]: - for env_value, value in [("true", True), ("false", False)]: - for name in get_boolean_options(): - env_var = "POETRY_{}".format(re.sub("[.-]+", "_", name).upper()) - yield name, env_var, env_value, value + for normalizer, values in [ + (boolean_normalizer, [("true", True), ("false", False)]), + (int_normalizer, [("4", 4), ("2", 2)]), + ]: + for env_value, value in values: + for name in get_options_based_on_normalizer(normalizer=normalizer): + env_var = "POETRY_{}".format(re.sub("[.-]+", "_", name).upper()) + yield name, env_var, env_value, value @pytest.mark.parametrize( From ed0b4a73d1633b448a9b934d757596dbb860acdd Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sat, 20 Nov 2021 15:03:23 +0100 Subject: [PATCH 09/11] Fix tests --- tests/console/commands/test_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/console/commands/test_config.py b/tests/console/commands/test_config.py index 8d9620b1b28..d4a2dfec857 100644 --- a/tests/console/commands/test_config.py +++ b/tests/console/commands/test_config.py @@ -46,6 +46,7 @@ def test_list_displays_default_value_if_not_set( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = true virtualenvs.in-project = null @@ -70,6 +71,7 @@ def test_list_displays_set_get_setting( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null @@ -118,6 +120,7 @@ def test_list_displays_set_get_local_setting( expected = """cache-dir = {cache} experimental.new-installer = true +installer.max-workers = null installer.parallel = true virtualenvs.create = false virtualenvs.in-project = null From 2e724ae8cf5e7e30efc93afc923eb2e4bab3a6dc Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 21 Nov 2021 16:08:30 +0100 Subject: [PATCH 10/11] Address PR comments --- poetry.lock | 13 ++++++++++++- pyproject.toml | 1 + tests/config/test_config.py | 28 ++++++---------------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/poetry.lock b/poetry.lock index e1987a36d8c..36b6f769f8b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -211,6 +211,14 @@ python-versions = ">=3.6" docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] +[[package]] +name = "flatdict" +version = "4.0.1" +description = "Python module for interacting with nested dicts as a single level dict with delimited keys." +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "html5lib" version = "1.1" @@ -739,7 +747,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "9d2e32899df46f2c63018e9a3f5e95dbbeb1ec41291c31289cff40f6f2d935a4" +content-hash = "d427df125a868ada92bbb6d3a8cc90def6034ad684c1546afb519729048ab150" [metadata.files] atomicwrites = [ @@ -929,6 +937,9 @@ filelock = [ {file = "filelock-3.4.0-py3-none-any.whl", hash = "sha256:2e139a228bcf56dd8b2274a65174d005c4a6b68540ee0bdbb92c76f43f29f7e8"}, {file = "filelock-3.4.0.tar.gz", hash = "sha256:93d512b32a23baf4cac44ffd72ccf70732aeff7b8050fcaf6d3ec406d954baf4"}, ] +flatdict = [ + {file = "flatdict-4.0.1.tar.gz", hash = "sha256:cd32f08fd31ed21eb09ebc76f06b6bd12046a24f77beb1fd0281917e47f26742"}, +] html5lib = [ {file = "html5lib-1.1-py2.py3-none-any.whl", hash = "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d"}, {file = "html5lib-1.1.tar.gz", hash = "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f"}, diff --git a/pyproject.toml b/pyproject.toml index 4ff1a3f4420..0dc64b4877c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ deepdiff = "^5.0" httpretty = "^1.0" typing-extensions = { version = "^4.0.0", python = "<3.8" } zipp = { version = "^3.4", python = "<3.8" } +flatdict = "^4.0.1" [tool.poetry.scripts] poetry = "poetry.console.application:main" diff --git a/tests/config/test_config.py b/tests/config/test_config.py index a1c662c305f..5a9e9477318 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -2,15 +2,14 @@ import re from typing import TYPE_CHECKING -from typing import Any from typing import Callable -from typing import Dict from typing import Iterator -from typing import Optional from typing import Tuple import pytest +from flatdict import FlatDict + from poetry.config.config import Config from poetry.config.config import boolean_normalizer from poetry.config.config import int_normalizer @@ -20,24 +19,9 @@ from pathlib import Path -def flatten(config: Optional[Dict[str, Any]], parent_key: str = "", sep="."): - items = [] - for k, v in config.items(): - new_key = parent_key + sep + k if parent_key else k - if isinstance(v, dict): - items.extend(flatten(v, new_key, sep=sep).items()) - else: - items.append((new_key, v)) - return dict(items) - - -def get_options_based_on_normalizer( - normalizer: Callable, config: Optional[Config] = None -) -> str: - if config is None: - config = Config() - - flattened_config = flatten(config.default_config) +def get_options_based_on_normalizer(normalizer: Callable) -> str: + config = Config() + flattened_config = FlatDict(config.default_config, delimiter=".") for k in flattened_config: if config._get_normalizer(k) == normalizer: @@ -64,7 +48,7 @@ def generate_environment_variable_tests() -> Iterator[Tuple[str, str, str, bool] ]: for env_value, value in values: for name in get_options_based_on_normalizer(normalizer=normalizer): - env_var = "POETRY_{}".format(re.sub("[.-]+", "_", name).upper()) + env_var = "POETRY_" + re.sub("[.-]+", "_", name).upper() yield name, env_var, env_value, value From d05ef885803eb3edc896eb8941cf429f73ee8802 Mon Sep 17 00:00:00 2001 From: Maximilian Speicher Date: Sun, 21 Nov 2021 20:56:59 +0100 Subject: [PATCH 11/11] Make _get_normalizer static --- src/poetry/config/config.py | 3 ++- tests/config/test_config.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/poetry/config/config.py b/src/poetry/config/config.py index 011a556c8ff..720e479f48f 100644 --- a/src/poetry/config/config.py +++ b/src/poetry/config/config.py @@ -133,7 +133,8 @@ def process(self, value: Any) -> Any: return re.sub(r"{(.+?)}", lambda m: self.get(m.group(1)), value) - def _get_normalizer(self, name: str) -> Callable: + @staticmethod + def _get_normalizer(name: str) -> Callable: if name in { "virtualenvs.create", "virtualenvs.in-project", diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 5a9e9477318..0f8c237c27d 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -20,11 +20,10 @@ def get_options_based_on_normalizer(normalizer: Callable) -> str: - config = Config() - flattened_config = FlatDict(config.default_config, delimiter=".") + flattened_config = FlatDict(Config.default_config, delimiter=".") for k in flattened_config: - if config._get_normalizer(k) == normalizer: + if Config._get_normalizer(k) == normalizer: yield k