Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

config: add support for installer.parallel #3088

Merged
merged 1 commit into from
Oct 23, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions docs/docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,15 @@ Defaults to one of the following directories:
- Windows: `C:\Users\<username>\AppData\Local\pypoetry\Cache`
- Unix: `~/.cache/pypoetry`

### `installer.parallel`: boolean

Use parallel execution when using the new (`>=1.1.0`) installer.
Defaults to `true`.

!!!note:
This configuration will be ignored, and parallel execution disabled when running
Python 2.7 under Windows.

### `virtualenvs.create`: boolean

Create a new virtual environment if one doesn't already exist.
Expand Down
13 changes: 11 additions & 2 deletions poetry/config/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class Config(object):
"path": os.path.join("{cache-dir}", "virtualenvs"),
},
"experimental": {"new-installer": True},
"installer": {"parallel": True},
}

def __init__(
Expand Down Expand Up @@ -131,14 +132,22 @@ def process(self, value): # type: (Any) -> Any
return re.sub(r"{(.+?)}", lambda m: self.get(m.group(1)), value)

def _get_validator(self, name): # type: (str) -> Callable
if name in {"virtualenvs.create", "virtualenvs.in-project"}:
if name in {
"virtualenvs.create",
"virtualenvs.in-project",
"installer.parallel",
}:
return boolean_validator

if name == "virtualenvs.path":
return str

def _get_normalizer(self, name): # type: (str) -> Callable
if name in {"virtualenvs.create", "virtualenvs.in-project"}:
if name in {
"virtualenvs.create",
"virtualenvs.in-project",
"installer.parallel",
}:
return boolean_normalizer

if name == "virtualenvs.path":
Expand Down
1 change: 1 addition & 0 deletions poetry/console/commands/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def unique_config_values(self):
boolean_normalizer,
True,
),
"installer.parallel": (boolean_validator, boolean_normalizer, True,),
}

return unique_config_values
Expand Down
5 changes: 4 additions & 1 deletion poetry/installation/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@


class Executor(object):
def __init__(self, env, pool, config, io, parallel=True):
def __init__(self, env, pool, config, io, parallel=None):
self._env = env
self._io = io
self._dry_run = False
Expand All @@ -42,6 +42,9 @@ def __init__(self, env, pool, config, io, parallel=True):
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env)

if parallel is None:
parallel = config.get("installer.parallel", True)

if parallel and not (PY2 and WINDOWS):
# This should be directly handled by ThreadPoolExecutor
# however, on some systems the number of CPUs cannot be determined
Expand Down
27 changes: 20 additions & 7 deletions tests/config/test_config.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,29 @@
import os

import pytest

def test_config_get_default_value(config):
assert config.get("virtualenvs.create") is True

@pytest.mark.parametrize(
("name", "value"), [("installer.parallel", True), ("virtualenvs.create", True)]
)
def test_config_get_default_value(config, name, value):
assert config.get(name) is value


def test_config_get_processes_depended_on_values(config):
assert os.path.join("/foo", "virtualenvs") == config.get("virtualenvs.path")


def test_config_get_from_environment_variable(config, environ):
assert config.get("virtualenvs.create")

os.environ["POETRY_VIRTUALENVS_CREATE"] = "false"
assert not config.get("virtualenvs.create")
@pytest.mark.parametrize(
("name", "env_value", "value"),
[
("installer.parallel", "true", True),
("installer.parallel", "false", False),
("virtualenvs.create", "true", True),
("virtualenvs.create", "false", False),
],
)
def test_config_get_from_environment_variable(config, environ, name, env_value, value):
env_var = "POETRY_{}".format("_".join(k.upper() for k in name.split(".")))
os.environ[env_var] = env_value
assert config.get(name) is value
27 changes: 27 additions & 0 deletions tests/console/commands/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from poetry.config.config_source import ConfigSource
from poetry.core.pyproject import PyProjectException
from poetry.factory import Factory
from poetry.utils._compat import PY2
from poetry.utils._compat import WINDOWS


@pytest.fixture()
Expand All @@ -28,6 +30,7 @@ def test_list_displays_default_value_if_not_set(tester, config):

expected = """cache-dir = "/foo"
experimental.new-installer = true
installer.parallel = true
virtualenvs.create = true
virtualenvs.in-project = null
virtualenvs.path = {path} # /foo{sep}virtualenvs
Expand All @@ -45,6 +48,7 @@ def test_list_displays_set_get_setting(tester, config):

expected = """cache-dir = "/foo"
experimental.new-installer = true
installer.parallel = true
virtualenvs.create = false
virtualenvs.in-project = null
virtualenvs.path = {path} # /foo{sep}virtualenvs
Expand Down Expand Up @@ -84,6 +88,7 @@ def test_list_displays_set_get_local_setting(tester, config):

expected = """cache-dir = "/foo"
experimental.new-installer = true
installer.parallel = true
virtualenvs.create = false
virtualenvs.in-project = null
virtualenvs.path = {path} # /foo{sep}virtualenvs
Expand Down Expand Up @@ -119,3 +124,25 @@ def test_set_cert(tester, auth_config_source, mocker):
tester.execute("certificates.foo.cert path/to/ca.pem")

assert "path/to/ca.pem" == auth_config_source.config["certificates"]["foo"]["cert"]


def test_config_installer_parallel(tester, command_tester_factory):
serial_enforced = PY2 and WINDOWS

tester.execute("--local installer.parallel")
assert tester.io.fetch_output().strip() == "true"

workers = command_tester_factory(
"install"
)._command._installer._executor._max_workers
assert workers > 1 or (serial_enforced and workers == 1)

tester.io.clear_output()
tester.execute("--local installer.parallel false")
tester.execute("--local installer.parallel")
assert tester.io.fetch_output().strip() == "false"

workers = command_tester_factory(
"install"
)._command._installer._executor._max_workers
assert workers == 1