From ad72055a298e5c065efae2b8a000239293f84b31 Mon Sep 17 00:00:00 2001 From: inimaz <93inigo93@gmail.com> Date: Mon, 3 Jun 2024 20:51:38 +0200 Subject: [PATCH] Allow_multiple_runs parameter --- codecarbon/emissions_tracker.py | 20 ++++++++++++-------- codecarbon/lock.py | 4 +++- examples/logging_to_file_exclusive_run.py | 4 ++-- pyproject.toml | 4 ++-- tests/test_config.py | 14 ++++++++++++-- 5 files changed, 31 insertions(+), 15 deletions(-) diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 15512d67c..8c5a6580d 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -162,7 +162,7 @@ def __init__( logger_preamble: Optional[str] = _sentinel, default_cpu_power: Optional[int] = _sentinel, pue: Optional[int] = _sentinel, - prevent_multiple_runs: Optional[bool] = _sentinel, + allow_multiple_runs: Optional[bool] = _sentinel, ): """ :param project_name: Project name for current experiment run, default name @@ -215,13 +215,17 @@ def __init__( messages. Defaults to "". :param default_cpu_power: cpu power to be used as default if the cpu is not known. :param pue: PUE (Power Usage Effectiveness) of the datacenter. - :param prevent_multiple_runs: Prevent multiple instances of codecarbon running. Defaults to False. + :param allow_multiple_runs: Allow multiple instances of codecarbon running in parallel. Defaults to False. """ # logger.info("base tracker init") self._external_conf = get_hierarchical_config() - self._set_from_conf(prevent_multiple_runs, "prevent_multiple_runs", False, bool) - if self._prevent_multiple_runs: + self._set_from_conf(allow_multiple_runs, "allow_multiple_runs", False, bool) + if self._allow_multiple_runs: + logger.warning( + "Multiple instances of codecarbon are allowed to run at the same time." + ) + else: # Acquire lock file to prevent multiple instances of codecarbon running # at the same time acquire_lock() @@ -945,7 +949,7 @@ def track_emissions( log_level: Optional[Union[int, str]] = _sentinel, default_cpu_power: Optional[int] = _sentinel, pue: Optional[int] = _sentinel, - prevent_multiple_runs: Optional[bool] = _sentinel, + allow_multiple_runs: Optional[bool] = _sentinel, ): """ Decorator that supports both `EmissionsTracker` and `OfflineEmissionsTracker` @@ -989,7 +993,7 @@ def track_emissions( Defaults to "info". :param default_cpu_power: cpu power to be used as default if the cpu is not known. :param pue: PUE (Power Usage Effectiveness) of the datacenter. - :param prevent_multiple_runs: Prevent multiple instances of codecarbon running. Defaults to False. + :param allow_multiple_runs: Prevent multiple instances of codecarbon running. Defaults to False. :return: The decorated function """ @@ -1022,7 +1026,7 @@ def wrapped_fn(*args, **kwargs): co2_signal_api_token=co2_signal_api_token, default_cpu_power=default_cpu_power, pue=pue, - prevent_multiple_runs=prevent_multiple_runs, + allow_multiple_runs=allow_multiple_runs, ) else: tracker = EmissionsTracker( @@ -1046,7 +1050,7 @@ def wrapped_fn(*args, **kwargs): co2_signal_api_token=co2_signal_api_token, default_cpu_power=default_cpu_power, pue=pue, - prevent_multiple_runs=prevent_multiple_runs, + allow_multiple_runs=allow_multiple_runs, ) tracker.start() try: diff --git a/codecarbon/lock.py b/codecarbon/lock.py index f72c21aa1..3343b1941 100644 --- a/codecarbon/lock.py +++ b/codecarbon/lock.py @@ -8,10 +8,12 @@ import errno import os import sys +import tempfile from codecarbon.external.logger import logger -LOCKFILE = "/tmp/.codecarbon.lock" +# We use tempfile.gettempdir() to get the system's temporary directory (linux: /tmp, windows: C:\Users\username\AppData\Local\Temp) +LOCKFILE = os.path.join(tempfile.gettempdir(), ".codecarbon.lock") lock_file_created_by_this_process = False diff --git a/examples/logging_to_file_exclusive_run.py b/examples/logging_to_file_exclusive_run.py index 9642fe294..7050f6fc0 100644 --- a/examples/logging_to_file_exclusive_run.py +++ b/examples/logging_to_file_exclusive_run.py @@ -1,5 +1,5 @@ """ -Similar to logging_to_file.py, but with the `prevent_multiple_runs` parameter set to True. +Similar to logging_to_file.py, but with the `allow_multiple_runs` parameter set to False (which is the default). This will prevent multiple instances of codecarbon from running at the same time. We run 5 instances of codecarbon. 4 wil fail and only one will succeed @@ -28,7 +28,7 @@ def worker(): country_iso_code="FRA", measure_power_secs=30, project_name="ultra_secret", - prevent_multiple_runs=True, + # allow_multiple_runs=True, # Set this to True to allow multiple instances of codecarbon to run at the same time ) tracker.start() diff --git a/pyproject.toml b/pyproject.toml index f9548bc8d..d176de41e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,8 +109,8 @@ dependencies = [ ] [tool.hatch.envs.test.scripts] -package = 'pytest -vv -m "not integ_test" tests/' -package-integ = 'python -m pytest -vv tests/' +package = 'CODECARBON_ALLOW_MULTIPLE_RUNS=True pytest -vv -m "not integ_test" tests/' +package-integ = 'CODECARBON_ALLOW_MULTIPLE_RUNS=True python -m pytest -vv tests/' [[tool.hatch.envs.test.matrix]] # Python 3.7 does not works with pip-tools diff --git a/tests/test_config.py b/tests/test_config.py index b15023ba2..814db22c1 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -55,7 +55,13 @@ def test_parse_gpu_ids(self): def test_parse_env_config(self): self.assertDictEqual( parse_env_config(), - {"codecarbon": {"test": "test-VALUE", "test_key": "this_other_value"}}, + { + "codecarbon": { + "allow_multiple_runs": "True", + "test": "test-VALUE", + "test_key": "this_other_value", + } + }, ) def test_read_confs(self): @@ -80,6 +86,7 @@ def test_read_confs(self): ): conf = dict(get_hierarchical_config()) target = { + "allow_multiple_runs": "True", "no_overwrite": "path/to/somewhere", "local_overwrite": "SUCCESS:overwritten", "syntax_test_key": "no/space= problem2", @@ -119,6 +126,7 @@ def test_read_confs_and_parse_envs(self): ): conf = dict(get_hierarchical_config()) target = { + "allow_multiple_runs": "True", "no_overwrite": "path/to/somewhere", "local_overwrite": "SUCCESS:overwritten", "env_overwrite": "SUCCESS:overwritten", @@ -136,7 +144,9 @@ def test_empty_conf(self): "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) ): conf = dict(get_hierarchical_config()) - target = {} + target = { + "allow_multiple_runs": "True" + } # allow_multiple_runs is a default value self.assertDictEqual(conf, target) @mock.patch.dict(