From e3f6cf77f2e81bd97db96234b79c968227429143 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Wed, 23 Oct 2024 12:01:04 +0200 Subject: [PATCH 1/2] build binary wheels But don't publish them yet --- .circleci/config.yml | 104 +++++++++++++++++++++++++ .github/workflows/ci-tests.yml | 16 ++-- .github/workflows/wheels.yml | 130 +++++++++++++++++++++++++++++++ MANIFEST.in | 1 + cibw-requirements.txt | 1 + cwltool/software_requirements.py | 2 + pyproject.toml | 13 ++++ test-requirements.txt | 2 +- tests/test_dependencies.py | 10 +-- 9 files changed, 265 insertions(+), 14 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .github/workflows/wheels.yml create mode 100644 cibw-requirements.txt diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000000..2127e18be3 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,104 @@ +version: 2.1 + +parameters: + REF: + type: string + default: '' + description: Optional tag to build + +jobs: + arm-wheels: + parameters: + build: + type: string + image: + type: string + + machine: + image: ubuntu-2204:current + resource_class: arm.medium # two vCPUs + + environment: + CIBW_ARCHS: "aarch64" + CIBW_MANYLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_MUSLLINUX_AARCH64_IMAGE: "<< parameters.image >>" + CIBW_BUILD: "<< parameters.build >>" + + steps: + - checkout + - when: + condition: << pipeline.parameters.REF >> + steps: + - run: + name: Checkout branch/tag << pipeline.parameters.REF >> + command: | + echo "Switching to branch/tag << pipeline.parameters.REF >> if it exists" + git checkout << pipeline.parameters.REF >> || true + git pull origin << pipeline.parameters.REF >> || true + - run: + name: install cibuildwheel and other build reqs + command: | + python3 -m pip install --upgrade pip setuptools setuptools_scm[toml] + python3 -m pip install -rcibw-requirements.txt + + - run: + name: pip freeze + command: | + python3 -m pip freeze + + - run: + name: list wheels + command: | + python3 -m cibuildwheel . --print-build-identifiers + + - run: + name: cibuildwheel + command: | + python3 -m cibuildwheel . + + - store_test_results: + path: test-results/ + + - store_artifacts: + path: wheelhouse/ + + # - when: + # condition: + # or: + # - matches: + # pattern: ".+" + # value: "<< pipeline.git.tag >>" + # - << pipeline.parameters.REF >> + # steps: + # - run: + # environment: + # TWINE_NONINTERACTIVE: "1" + # command: | + # python3 -m pip install twine + # python3 -m twine upload --verbose --skip-existing wheelhouse/* + +workflows: + wheels: # This is the name of the workflow, feel free to change it to better match your workflow. + # Inside the workflow, you define the jobs you want to run. + jobs: + - arm-wheels: + name: arm-wheels-manylinux_2_28 + filters: + tags: + only: /.*/ + build: "*manylinux*" + image: quay.io/pypa/manylinux_2_28_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_1 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_1_aarch64 + - arm-wheels: + name: arm-wheels-musllinux_1_2 + filters: + tags: + only: /.*/ + build: "*musllinux*" + image: quay.io/pypa/musllinux_1_2_aarch64 diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2095b53b64..e5d9a7e839 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -44,11 +44,11 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. if: ${{ matrix.step == 'unit' || matrix.step == 'mypy' }} @@ -132,10 +132,10 @@ jobs: with: fetch-depth: 0 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-focal_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-focal_amd64.deb environment-modules - name: Give the test runner user a name to make provenance happy. run: sudo usermod -c 'CI Runner' "$(whoami)" @@ -180,11 +180,11 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules if: ${{ matrix.container == 'singularity' }} run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Singularity cache if: ${{ matrix.container == 'singularity' }} @@ -229,10 +229,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Singularity + - name: Set up Singularity and environment-modules run: | wget --no-verbose https://github.com/sylabs/singularity/releases/download/v3.10.4/singularity-ce_3.10.4-jammy_amd64.deb - sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb + sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb environment-modules - name: Set up Python uses: actions/setup-python@v5 diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000000..9c14eb4e76 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,130 @@ +name: Python package build and publish + +on: + release: + types: [published] + workflow_dispatch: {} + repository_dispatch: {} + pull_request: + push: + branches: + - main + +concurrency: + group: wheels-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build_wheels: + name: ${{ matrix.image }} wheels + runs-on: ubuntu-24.04 + strategy: + matrix: + include: + - image: manylinux_2_28_x86_64 + build: "*manylinux*" + - image: musllinux_1_1_x86_64 + build: "*musllinux*" + - image: musllinux_1_2_x86_64 + build: "*musllinux*" + + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + # - name: Set up QEMU + # if: runner.os == 'Linux' + # uses: docker/setup-qemu-action@v2 + # with: + # platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + env: + CIBW_BUILD: ${{ matrix.build }} + CIBW_MANYLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + CIBW_MUSLLINUX_X86_64_IMAGE: quay.io/pypa/${{ matrix.image }} + # configure cibuildwheel to build native 64-bit archs ('auto64'), and some + # emulated ones + # Linux arm64 wheels are built on circleci + CIBW_ARCHS_LINUX: auto64 # ppc64le s390x + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.image }} + path: ./wheelhouse/*.whl + + build_sdist: + name: Build source distribution + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build sdist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v4 + with: + name: artifact-source + path: dist/*.tar.gz + + build_wheels_macos: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + # macos-13 is an intel runner, macos-14 is apple silicon + os: [macos-13, macos-14] + steps: + - uses: actions/checkout@v4 + if: ${{ github.event_name != 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + - uses: actions/checkout@v4 + if: ${{ github.event_name == 'repository_dispatch' }} + with: + fetch-depth: 0 # slow, but gets all the tags + ref: ${{ github.event.client_payload.ref }} + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21.3 + + - uses: actions/upload-artifact@v4 + with: + name: artifact-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + # upload_pypi: + # needs: [build_wheels, build_sdist] + # runs-on: ubuntu-24.04 + # environment: deploy + # permissions: + # id-token: write + # if: (github.event_name == 'release' && github.event.action == 'published') || (github.event_name == 'repository_dispatch' && github.event.client_payload.publish_wheel == true) + # steps: + # - uses: actions/download-artifact@v4 + # with: + # # unpacks default artifact into dist/ + # pattern: artifact-* + # merge-multiple: true + # path: dist + + # - name: Publish package distributions to PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # skip-existing: true diff --git a/MANIFEST.in b/MANIFEST.in index 187d19bea1..7ee34f35ee 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -19,6 +19,7 @@ include tests/reloc/dir2/* include tests/checker_wf/* include tests/subgraph/* include tests/input_deps/* +recursive-include tests/test_deps_env include tests/trs/* include tests/wf/generator/* include cwltool/py.typed diff --git a/cibw-requirements.txt b/cibw-requirements.txt new file mode 100644 index 0000000000..c4511439cc --- /dev/null +++ b/cibw-requirements.txt @@ -0,0 +1 @@ +cibuildwheel==2.21.3 diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index 6ad84da4bd..3d4d48f6b1 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -50,6 +50,8 @@ class DependenciesConfiguration: def __init__(self, args: argparse.Namespace) -> None: """Initialize.""" + self.tool_dependency_dir: Optional[str] = None + self.dependency_resolvers_config_file: Optional[str] = None conf_file = getattr(args, "beta_dependency_resolvers_configuration", None) tool_dependency_dir = getattr(args, "beta_dependencies_directory", None) conda_dependencies = getattr(args, "beta_conda_dependencies", None) diff --git a/pyproject.toml b/pyproject.toml index cec213f521..a69720739b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,19 @@ build-backend = "setuptools.build_meta" [tool.setuptools_scm] write_to = "cwltool/_version.py" +[tool.cibuildwheel] +test-command = "python -m pytest -n 2 --junitxml={project}/test-results/junit_$(python -V | awk '{print $2}')_${AUDITWHEEL_PLAT}.xml -k 'not (test_bioconda or test_env_filtering or test_udocker)' --pyargs cwltool" +test-requires = "-r test-requirements.txt" +test-extras = "deps" +skip = "pp*" +# ^ skip building wheels on PyPy (any version) +build-verbosity = 1 +environment = { CWLTOOL_USE_MYPYC="1", MYPYPATH="$(pwd)/mypy-stubs" } + +# Install system library +[tool.cibuildwheel.linux] +before-all = "apk add libxml2-dev libxslt-dev nodejs || yum install -y libxml2-devel libxslt-devel nodejs environment-modules || apt-get install -y --no-install-recommends libxml2-dev libxslt-dev nodejs environment-modules" + [tool.black] line-length = 100 target-version = [ "py39" ] diff --git a/test-requirements.txt b/test-requirements.txt index e545ee65a6..8b0908f2e2 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,7 +3,7 @@ pytest>= 6.2,< 8.4 pytest-xdist>=3.2.0 # for the worksteal scheduler psutil # enhances pytest-xdist to allow "-n logical" pytest-httpserver -pytest-retry;python_version>'3.9' +pytest-retry;python_version>='3.9' mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index b903c04d6a..f5ac0274bc 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -119,15 +119,15 @@ def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement.""" wflow = get_data("tests/random_lines.cwl") job = get_data("tests/random_lines_job.json") - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) error_code, _, stderr = get_main_output( [ "--outdir", str(tmp_path / "out"), - "--beta-dependency-resolvers-configuration", "--beta-dependencies-directory", str(tmp_path / "deps"), - "tests/test_deps_env_modules_resolvers_conf.yml", + "--beta-dependency-resolvers-configuration", + get_data("tests/test_deps_env_modules_resolvers_conf.yml"), "--debug", wflow, job, @@ -145,7 +145,7 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Do so by by running `env` as the tool and parsing its output. """ - monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles")) + monkeypatch.setenv("MODULEPATH", get_data("tests/test_deps_env/modulefiles")) tool_env = get_tool_env( tmp_path, [ @@ -155,6 +155,6 @@ def test_modules_environment(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> get_data("tests/env_with_software_req.yml"), ) - assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space " + assert tool_env["TEST_VAR_MODULE"] == "environment variable ends in space ", tool_env tool_path = tool_env["PATH"].split(":") assert get_data("tests/test_deps_env/random-lines/1.0/scripts") in tool_path From 6be8e4d60b986aa001d12591366bd0504258f4fa Mon Sep 17 00:00:00 2001 From: suecharo Date: Sat, 13 Apr 2019 13:04:57 +0900 Subject: [PATCH 2/2] Factory: also parse command-line options Co-authored-by: Michael R. Crusoe --- cwltool/factory.py | 87 +++++++++++++++++++++++++++++++++++------- tests/test_context.py | 2 +- tests/test_parallel.py | 4 +- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/cwltool/factory.py b/cwltool/factory.py index eaf98e3cfa..fd47d77b64 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,12 +1,21 @@ +"""Wrap a CWL document as a callable Python object.""" + +import argparse +import functools import os +import sys from typing import Any, Optional, Union from . import load_tool -from .context import LoadingContext, RuntimeContext +from .argparser import arg_parser +from .context import LoadingContext, RuntimeContext, getdefault from .errors import WorkflowException from .executors import JobExecutor, SingleJobExecutor +from .main import find_default_container from .process import Process -from .utils import CWLObjectType +from .resolver import tool_resolver +from .secrets import SecretStore +from .utils import DEFAULT_TMP_PREFIX, CWLObjectType class WorkflowStatus(Exception): @@ -25,11 +34,15 @@ def __init__(self, t: Process, factory: "Factory") -> None: self.t = t self.factory = factory - def __call__(self, **kwargs): - # type: (**Any) -> Union[str, Optional[CWLObjectType]] - runtime_context = self.factory.runtime_context.copy() - runtime_context.basedir = os.getcwd() - out, status = self.factory.executor(self.t, kwargs, runtime_context) + def __call__(self, **kwargs: Any) -> Union[str, Optional[CWLObjectType]]: + """ + Execute the process. + + :raise WorkflowStatus: If the result is not a success. + """ + if not self.factory.runtime_context.basedir: + self.factory.runtime_context.basedir = os.getcwd() + out, status = self.factory.executor(self.t, kwargs, self.factory.runtime_context) if status != "success": raise WorkflowStatus(out, status) else: @@ -47,18 +60,24 @@ def __init__( executor: Optional[JobExecutor] = None, loading_context: Optional[LoadingContext] = None, runtime_context: Optional[RuntimeContext] = None, + argsl: Optional[list[str]] = None, + args: Optional[argparse.Namespace] = None, ) -> None: + """Create a CWL Process factory from a CWL document.""" + if argsl is not None: + args = arg_parser().parse_args(argsl) if executor is None: - executor = SingleJobExecutor() - self.executor = executor + self.executor: JobExecutor = SingleJobExecutor() + else: + self.executor = executor if runtime_context is None: - self.runtime_context = RuntimeContext() + self.runtime_context = RuntimeContext(vars(args) if args else {}) + self._fix_runtime_context() else: self.runtime_context = runtime_context if loading_context is None: - self.loading_context = LoadingContext() - self.loading_context.singularity = self.runtime_context.singularity - self.loading_context.podman = self.runtime_context.podman + self.loading_context = LoadingContext(vars(args) if args else {}) + self._fix_loading_context(self.runtime_context) else: self.loading_context = loading_context @@ -68,3 +87,45 @@ def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: if isinstance(load, int): raise WorkflowException("Error loading tool") return Callable(load, self) + + def _fix_loading_context(self, runtime_context: RuntimeContext) -> None: + self.loading_context.resolver = getdefault(self.loading_context.resolver, tool_resolver) + self.loading_context.singularity = runtime_context.singularity + self.loading_context.podman = runtime_context.podman + + def _fix_runtime_context(self) -> None: + self.runtime_context.basedir = os.getcwd() + self.runtime_context.find_default_container = functools.partial( + find_default_container, default_container=None, use_biocontainers=None + ) + + if sys.platform == "darwin": + default_mac_path = "/private/tmp/docker_tmp" + if self.runtimeContext.tmp_outdir_prefix == DEFAULT_TMP_PREFIX: + self.runtimeContext.tmp_outdir_prefix = default_mac_path + + for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): + if ( + getattr(self.runtime_context, dirprefix) + and getattr(self.runtime_context, dirprefix) != DEFAULT_TMP_PREFIX + ): + sl = ( + "/" + if getattr(self.runtime_context, dirprefix).endswith("/") + or dirprefix == "cachedir" + else "" + ) + setattr( + self.runtime_context, + dirprefix, + os.path.abspath(getattr(self.runtime_context, dirprefix)) + sl, + ) + if not os.path.exists(os.path.dirname(getattr(self.runtime_context, dirprefix))): + try: + os.makedirs(os.path.dirname(getattr(self.runtime_context, dirprefix))) + except Exception as e: + print("Failed to create directory: %s", e) + + self.runtime_context.secret_store = getdefault( + self.runtime_context.secret_store, SecretStore() + ) diff --git a/tests/test_context.py b/tests/test_context.py index acb7dfbcad..6c37f062da 100644 --- a/tests/test_context.py +++ b/tests/test_context.py @@ -20,7 +20,7 @@ def test_replace_default_stdout_stderr() -> None: runtime_context = RuntimeContext() runtime_context.default_stdout = subprocess.DEVNULL # type: ignore runtime_context.default_stderr = subprocess.DEVNULL # type: ignore - factory = Factory(None, None, runtime_context) + factory = Factory(runtime_context=runtime_context) echo = factory.make(get_data("tests/echo.cwl")) assert echo(inp="foo") == {"out": "foo\n"} diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 8c86d41fc1..3d63515ea6 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -15,7 +15,7 @@ def test_sequential_workflow(tmp_path: Path) -> None: runtime_context = RuntimeContext() runtime_context.outdir = str(tmp_path) runtime_context.select_resources = executor.select_resources - factory = Factory(executor, None, runtime_context) + factory = Factory(executor=executor, runtime_context=runtime_context) echo = factory.make(get_data(test_file)) file_contents = {"class": "File", "location": get_data("tests/wf/whale.txt")} assert echo(file1=file_contents) == {"count_output": 16} @@ -25,7 +25,7 @@ def test_sequential_workflow(tmp_path: Path) -> None: def test_scattered_workflow() -> None: test_file = "tests/wf/scatter-wf4.cwl" job_file = "tests/wf/scatter-job2.json" - factory = Factory(MultithreadedJobExecutor()) + factory = Factory(executor=MultithreadedJobExecutor()) echo = factory.make(get_data(test_file)) with open(get_data(job_file)) as job: assert echo(**json.load(job)) == {"out": ["foo one three", "foo two four"]}