Skip to content

Commit

Permalink
Rework utopya test invocation to support process spawning
Browse files Browse the repository at this point in the history
The pytest framework seems to have its troubles with multiprocessing.
Prior to this commit, running `test_task.py` individually would succeed,
while running the whole test suite would fail, just as one example.

This commit addresses those issues by wrapping `pytest.main` execution
into the `if __name__ == '__main__'` guard and ensuring that the utopya
module is in the system path (regardless of how pytest may have modified
it). See pytest-dev/pytest#958 for more info.
  • Loading branch information
blsqr committed Dec 18, 2020
1 parent 74541c3 commit 910ce7a
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 5 deletions.
23 changes: 23 additions & 0 deletions run_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/usr/bin/python3
"""This is a custom entrypoint to pytest which takes care that pytest is
run within the ``if __name__ == '__main__'`` guard, thereby resolving issues
with the multiprocessing module.
We are not the only ones with this problem, see:
- https://github.com/pytest-dev/pytest/issues/958
- https://github.com/web-platform-tests/wpt/issues/24880
"""

import os
import sys

import pytest

# Ensure that utopya is in the path and already imported at this point. If not
# doing this, discovery of the utopya module may fail if the test modules are
# imported when *spawning* a new multiprocessing.Process ...
sys.path.insert(0, os.path.dirname(__file__))
import utopya

if __name__ == '__main__':
sys.exit(pytest.main([__file__] + sys.argv[1:]))
7 changes: 3 additions & 4 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@
'ruamel.yaml>=0.16.5',

'paramspace>=2.5.4',
'dantro>=0.15.0a2',
'dantro>=0.15.2',
# NOTE Version need also be set in python/CMakeLists.txt

# Required for testing:
'pytest>=3.4.0',
'pytest-cov>=2.5.1',
'pytest>=6.2.1',
'pytest-cov>=2.10.1',
'psutil>=5.6.7'
]

Expand Down Expand Up @@ -70,7 +70,6 @@ def read(*parts):
packages=find_packages(exclude=["*.test", "*.test.*", "test.*", "test"]),
package_data=dict(utopya=["cfg/*.yml"]),
install_requires=INSTALL_DEPS,
test_suite='py.test',
#
# Command line scripts, installed into the virtual environment
scripts=['bin/utopia']
Expand Down
2 changes: 1 addition & 1 deletion test/cfg/batch_file.yml
Original file line number Diff line number Diff line change
Expand Up @@ -61,5 +61,5 @@ tasks:
# All other arguments are passed to `Model.create_frozen_mv`

task2:
model_name: Allhoff2015
model_name: dummy
run_dir: ~
4 changes: 4 additions & 0 deletions test/test_task.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,13 @@
from time import sleep
from functools import partial

import multiprocessing
# NOTE Need this import, otherwise tests that use multiprocessing hang forever

import numpy as np
import pytest

import utopya
from utopya.task import (Task, WorkerTask, TaskList,
PopenMPProcess, MPProcessTask,
enqueue_lines, parse_yaml_dict, SIGMAP)
Expand Down

0 comments on commit 910ce7a

Please sign in to comment.