From 152d8026c05c6f11b8f189b85cec283165acd1d5 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Mon, 30 Sep 2024 20:09:51 +0200 Subject: [PATCH] chore: move CI to GitHub Actions (#1763) Move the CI from CircleCI to GitHub actions. As GitHub Actions work a bit differently, things needed to change a bit and I took the chance to clean things up a bit. Nonetheless the coverage should be the same. If you compare it to the CircleCI configuration and wonder how different things are, this list of changes might help: - Seperate multicore-sdr run was removed, it's enabled on the normal tests - Building benchmarks was removed as `--all-targets` already implies building the benchmarks - All the `test` jobs are removed, they are already covered by the release mode tests that run for the whole workspace - Some of the `ignored` tests were run separately. Now simply the whole workspace is run in a single job - The `no_gpu` test was renamed to `no-default-features` - `test_darwin` was renamed to `test_macos` - The `test_arm_no_gpu` wasn't ported as the MacOS runner is an M1, hence there is already basic ARM sanity checking Only generate proofs parameters every two months and not on every PR. --- .circleci/config.yml | 621 ------------------ .github/README.md | 12 + .../actions/proof-params-download/action.yml | 35 + .github/workflows/ci.yml | 153 +++++ .github/workflows/proof-params-generate.yml | 37 ++ 5 files changed, 237 insertions(+), 621 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/README.md create mode 100644 .github/actions/proof-params-download/action.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/proof-params-generate.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b8ef4e896..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,621 +0,0 @@ -version: 2.1 - -executors: - default: - docker: - - image: cimg/rust:1.70 - resource_class: 2xlarge+ - gpu: - machine: - image: linux-cuda-12:2023.05.1 - working_directory: ~/gpuci - resource_class: gpu.nvidia.medium - arm: - machine: - image: ubuntu-2004:202101-01 - resource_class: arm.large - -setup-env: &setup-env - # The sparse index makes the `cargo update` a lot faster. - CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse - FIL_PROOFS_PARAMETER_CACHE: "/tmp/filecoin-proof-parameters/" - RUST_LOG: info - - -jobs: - ensure_groth_parameters_and_keys_linux: - executor: default - environment: *setup-env - steps: - - checkout - - restore_parameter_cache - - ensure_filecoin_parameters - - save_parameter_cache - - cargo_fetch: - executor: default - resource_class: small - environment: *setup-env - steps: - - checkout - - run: - name: Calculate dependencies - command: cargo generate-lockfile - no_output_timeout: 30m - - restore_rustup_cache - - run: rustup install $(cat rust-toolchain) - - run: rustup default $(cat rust-toolchain) - - run: cargo update - - run: cargo fetch - - run: rustc +$(cat rust-toolchain) --version - - run: rustup toolchain list --verbose - - persist_to_workspace: - root: "." - paths: - - Cargo.lock - - save_rustup_cache - test: - executor: default - environment: *setup-env - parameters: - crate: - type: string - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Test (<< parameters.crate >>) - command: cargo test --verbose --package << parameters.crate >> - no_output_timeout: 30m - - test_release: - executor: default - environment: *setup-env - parameters: - cargo-args: - description: Addtional arguments for the cargo command - type: string - default: "" - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Test in release profile - command: | - ulimit -n 20000 - ulimit -u 20000 - ulimit -n 20000 - cargo test --verbose --release --workspace --all-targets << parameters.cargo-args >> - # Some `storage-proofs-update` tests need to run sequentially due - # to their high memory usage. - cargo test -p storage-proofs-update --features isolated-testing --release << parameters.cargo-args >> -- --test-threads=1 - no_output_timeout: 30m - - test_ignored_release: - executor: default - environment: *setup-env - parameters: - crate: - type: string - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Test ignored in release profile - command: | - ulimit -n 20000 - ulimit -u 20000 - ulimit -n 20000 - cd << parameters.crate >> - cargo test --release -- --ignored --nocapture - environment: - RUST_TEST_THREADS: 1 - no_output_timeout: 30m - - - # Running with `use_multicore_sdr=true` should be integrated directly into the test code. For now we - # just re-run the lifecycle tests to exercise the use_multicore_sdr code path with that setting set. - test_multicore_sdr: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Test with use_multicore_sdr - command: | - ulimit -n 20000 - ulimit -u 20000 - ulimit -n 20000 - cargo test --all --verbose --release lifecycle -- --ignored --nocapture - cargo test -p storage-proofs-porep --features isolated-testing --release --lib stacked::vanilla::cores - cargo test -p storage-proofs-porep --features isolated-testing --release test_parallel_generation_and_read_partial_range_v1_0 - cargo test -p storage-proofs-porep --features isolated-testing --release test_parallel_generation_and_read_partial_range_v1_1 - no_output_timeout: 30m - environment: - RUST_LOG: debug - RUST_TEST_THREADS: 1 - FIL_PROOFS_USE_MULTICORE_SDR: true - - test_gpu_tree_building: - executor: gpu - environment: *setup-env - parameters: - cargo-args: - description: Addtional arguments for the cargo command - type: string - default: "" - test-args: - description: Additional arguments for the test executable (after the `--`) - type: string - default: "" - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - run: - name: Set the PATH env variable - command: | - echo 'export PATH="~/.cargo/bin:/usr/local/cuda-11.2/bin:$PATH"' | tee --append $BASH_ENV - source $BASH_ENV - - install_libs - - run: - name: Test with GPU column and tree builders. - command: | - ulimit -n 20000 - ulimit -u 20000 - ulimit -n 20000 - cargo test --verbose --release << parameters.cargo-args >> -- --nocapture << parameters.test-args >> - no_output_timeout: 30m - environment: - FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true - FIL_PROOFS_USE_GPU_TREE_BUILDER: true - BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 - NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 - - test_release_supraseal: - executor: gpu - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Set the PATH env variable - command: | - echo 'export PATH="~/.cargo/bin:$PATH"' | tee --append $BASH_ENV - source $BASH_ENV - - run: - name: SupraSeal filecoin-proofs tests in release profile - command: | - cargo test -p filecoin-proofs --release --no-default-features --features cuda-supraseal -- --nocapture --test-threads=1 - cargo test -p filecoin-proofs --release --no-default-features --features cuda-supraseal -- --nocapture --test-threads=1 --ignored - - test_no_gpu: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - run: - name: Test with no gpu - command: | - cargo test --all --verbose --release --no-default-features - no_output_timeout: 30m - - test_arm_no_gpu: - executor: arm - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: rustup install $(cat rust-toolchain) - - run: rustup default $(cat rust-toolchain) - - run: cargo update - - run: cargo fetch - - install_libs - - run: - name: Test arm with no gpu - command: | - cargo test --release --all --verbose --no-default-features - no_output_timeout: 90m - - bench: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - restore_parameter_cache - - install_libs - - run: - name: Benchmarks - command: cargo build --benches --verbose --workspace - no_output_timeout: 15m - - rustfmt: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - run: - name: Run cargo fmt - command: cargo fmt --all -- --check - - clippy: - executor: default - environment: *setup-env - steps: - - checkout - - attach_workspace: - at: "." - - restore_rustup_cache - - install_libs - - run: - name: Run cargo clippy - command: cargo clippy --all-targets --workspace -- -D warnings - test_darwin: - macos: - xcode: "13.4.1" - working_directory: ~/crate - resource_class: macos.m1.medium.gen1 - environment: *setup-env - steps: - - run: - name: Delete simulators for more free disk space - command: sudo rm -rf /Library/Developer/CoreSimulator/Profiles/Runtimes/ - - checkout - - run: - name: Install hwloc with Homebrew - command: HOMEBREW_NO_AUTO_UPDATE=1 brew install hwloc - - run: - name: Install Rust - command: | - curl https://sh.rustup.rs -sSf | sh -s -- -y - - run: cargo update - - run: cargo fetch - # On CircleCI you cannot create a cache on Linux and restore it on macOS. - # Hence we need a separate cache for macOS that we create and restore. It - # is not a separate job (like `ensure_groth_parameters_and_keys_linux`, - # as there is only this single job that runs on macOS. - - restore_parameter_cache - - ensure_filecoin_parameters - - save_parameter_cache - - run: - name: Test Darwin - command: | - sudo ulimit -n 20000 - sudo ulimit -u 20000 - ulimit -n 20000 - cargo test --release --verbose --workspace -- --nocapture - no_output_timeout: 2h - - # Code coverage, using tarpaulin tool (https://github.com/xd009642/tarpaulin) - coverage-with-tarpaulin: - environment: *setup-env - machine: true - resource_class: 2xlarge - steps: - - checkout - - restore_parameter_cache - - run: - name: Pull xd009642/tarpaulin:latest - command: docker pull xd009642/tarpaulin:latest - - run: - name: Get coverage result - command: >- - docker run - --security-opt seccomp=unconfined - -v "${PWD}:/volume" - -v "${FIL_PROOFS_PARAMETER_CACHE}:/var/tmp/filecoin-proof-parameters/" - xd009642/tarpaulin - sh -c "apt update && apt install -y libssl-dev ocl-icd-opencl-dev libhwloc-dev && cargo tarpaulin --timeout 1800 --release -v" - no_output_timeout: 30m -commands: - ensure_filecoin_parameters: - steps: - - run: - name: Create parameters directory - command: mkdir -p ${FIL_PROOFS_PARAMETER_CACHE} - - run: - name: Build paramcache if it doesn't already exist - command: | - set -x; test -f /tmp/paramcache.awesome \ - || (cargo build --release --no-default-features --bin paramcache && find . -type f -name paramcache | xargs -I {} mv {} /tmp/paramcache.awesome) - - run: - name: Obtain filecoin groth parameters - command: /tmp/paramcache.awesome --sector-sizes='2048,4096,16384,32768' - no_output_timeout: 60m - - run: - name: Obtain filecoin ipp parameter file - command: | - # paramfetch is using `ipget` which currently always times out on - # CI, hence get this file via HTTP instead. - #cargo run --release --bin paramfetch -- -a -j srs-inner-product.json - # `--insecure` is needed due to an outdated base systems. - curl --insecure https://proofs.filecoin.io/v28-fil-inner-product-v1.srs --output ${FIL_PROOFS_PARAMETER_CACHE}v28-fil-inner-product-v1.srs - - run: - name: Make the parameters world readable - command: chmod -R 755 ${FIL_PROOFS_PARAMETER_CACHE} - save_rustup_cache: - steps: - # Move things from the home directory to `/tmp` first, so that it can be - # restored on executors that have a different home directory. - - run: cp -fR ~/.cargo ~/.rustup /tmp/ - - save_cache: - name: "Save rustup cache" - key: cargo-v28-e-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} - paths: - - /tmp/.cargo - - /tmp/.rustup - restore_rustup_cache: - steps: - - restore_cache: - name: "Restore rustup cache" - key: cargo-v28-e-{{ checksum "rust-toolchain" }}-{{ checksum "Cargo.toml" }}-{{ checksum "Cargo.lock" }} - # Cache might not be created yet, hence ignore if the move fails - - run: cp -R /tmp/.cargo /tmp/.rustup ~/ || true - save_parameter_cache: - steps: - # Make sure we have different cache for Linux and Darwin. - - run: uname > os.txt - - save_cache: - name: "Save parameter cache" - key: proof-params-v28-n-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }} - paths: - - "/tmp/paramcache.awesome" - - "/tmp/filecoin-proof-parameters/" - restore_parameter_cache: - steps: - # Make sure we have different cache for Linux and Darwin. - - run: uname > os.txt - - restore_cache: - name: "Restore parameter cache" - key: proof-params-v28-n-{{ checksum "os.txt" }}-{{ checksum "filecoin-proofs/parameters.json" }} - install_libs: - steps: - - run: - name: Install required libraries - command: | - sudo apt update - sudo apt install --yes ocl-icd-opencl-dev libhwloc-dev - -workflows: - version: 2.1 - test_all: - jobs: - - ensure_groth_parameters_and_keys_linux - - cargo_fetch - - rustfmt: - requires: - - cargo_fetch - - clippy: - requires: - - cargo_fetch - - - test_release: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_release: - name: test_release (fixed-rows-to-discard) - cargo-args: "--features fixed-rows-to-discard" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_post - crate: "storage-proofs-post" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_core - crate: "storage-proofs-core" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_storage_proofs_porep - crate: "storage-proofs-porep" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_ignored_release: - name: test_ignored_release_filecoin_proofs - crate: "filecoin-proofs" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_opencl (regular) - cargo-args: "--workspace" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_opencl (ignored, filecoin-proofs, storage-proofs-update) - cargo-args: "--package filecoin-proofs --package storage-proofs-update" - test-args: "--ignored" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_opencl (ignored, rest) - cargo-args: "--workspace --exclude filecoin-proofs --exclude storage-proofs-update" - test-args: "--ignored" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_cuda (regular) - cargo-args: "--workspace --features cuda" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_cuda (ignored, filecoin-proofs, storage-proofs-update) - cargo-args: "--package filecoin-proofs --package storage-proofs-update --features cuda" - test-args: "--ignored" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_gpu_tree_building: - name: test_gpu_tree_building_cuda (ignored, rest) - cargo-args: "--workspace --exclude filecoin-proofs --exclude storage-proofs-update --features cuda" - test-args: "--ignored" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_multicore_sdr: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_filecoin_proofs - crate: "filecoin-proofs" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_core - crate: "storage-proofs-core" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_post - crate: "storage-proofs-post" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_storage_proofs_porep - crate: "storage-proofs-porep" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_fil_proofs_tooling - crate: "fil-proofs-tooling" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_sha2raw - crate: "sha2raw" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test: - name: test_filecoin_hashers - crate: "filecoin-hashers" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - - test: - name: test_fil_proofs_param - crate: "fil-proofs-param" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_no_gpu: - name: test_no_gpu - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_arm_no_gpu: - name: test_arm_no_gpu - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - bench: - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_darwin - - - test: - name: test_fr32 - crate: "fr32" - requires: - - cargo_fetch - - - test: - name: test_fil_store_proofs_update - crate: "storage-proofs-update" - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - - - test_release_supraseal: - name: Test with cuda-supraseal feature enabled - requires: - - cargo_fetch - - ensure_groth_parameters_and_keys_linux - -# Temporarily disable due to failing permission issue -# - coverage-with-tarpaulin: -# requires: -# - ensure_groth_parameters_and_keys_linux diff --git a/.github/README.md b/.github/README.md new file mode 100644 index 000000000..f1d4092cd --- /dev/null +++ b/.github/README.md @@ -0,0 +1,12 @@ +GitHub Actions setup +==================== + +The main CI is happening in the [`ci.yml`], this is where the tests are run. + +There is a separate workflow called [`proof-params-generate.yml`], that pre-generates the Groth16 parameter files needed for testing. Those parameters are stored as GitHub Artifacts, which are then downloaded by any workflow that needs them. Those artifacts are available for 90 days. Hence this job runs every two months to regenerate them, but also to have enough time in case something breaks. + +The [`proof-params-download` Action] is a helper for downloading the pre-generated Groth16 parameters. + +[`ci.yml`]: ./workflows/ci.yml +[`proof-params-generate.yml`]: ./workflows/proof-params-generate.yml +[`proof-params-download` Action]: ./actions/proof-params-download/action.yml diff --git a/.github/actions/proof-params-download/action.yml b/.github/actions/proof-params-download/action.yml new file mode 100644 index 000000000..58fcaa217 --- /dev/null +++ b/.github/actions/proof-params-download/action.yml @@ -0,0 +1,35 @@ +name: Download proof params + +inputs: + github-token: + description: 'A secrets.GITHUB_TOKEN' + required: true + +runs: + using: 'composite' + steps: + - name: Get run-id from latest proof-params-generate workflow run + uses: octokit/request-action@v2.x + id: latest-proof-params + with: + route: GET /repos/filecoin-project/rust-fil-proofs/actions/workflows/proof-params-generate.yml/runs?per_page=1 + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + - name: Download all Artifacts from proof-params-generate workflow + uses: actions/download-artifact@v4 + with: + #pattern: proof-params-v28-n-${{ inputs.parameters-json-hash }}-* + pattern: proof-params-v28-n-${{ hashFiles('filecoin-proofs/parameters.json') }}-* + path: /var/tmp/filecoin-proof-parameters/ + merge-multiple: true + run-id: ${{ fromJson(steps.latest-proof-params.outputs.data).workflow_runs[0].id }} + github-token: ${{ inputs.github-token }} + - name: Obtain Filecoin IPP parameter file + shell: bash + # paramfetch is using `ipget` which currently always times out on CI, + # hence get this file via HTTP instead. + #cargo run --release --bin paramfetch -- -a -j srs-inner-product.json + run: 'curl https://proofs.filecoin.io/v28-fil-inner-product-v1.srs --output /var/tmp/filecoin-proof-parameters/v28-fil-inner-product-v1.srs' + - name: List parameter files + shell: bash + run: ls -al /var/tmp/filecoin-proof-parameters/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..a05d77bb9 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,153 @@ +name: CI + +on: [pull_request, push] + +# Cancel a job if there's a new on on the same branch started. +# Based on https://stackoverflow.com/questions/58895283/stop-already-running-workflow-job-in-github-actions/67223051#67223051 +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +env: + CARGO_INCREMENTAL: 0 + RUST_BACKTRACE: 1 + # Faster crates.io index checkout. + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + RUST_LOG: debug + +jobs: + check_clippy: + runs-on: ubuntu-24.04 + name: Clippy + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev + - name: Run cargo clippy + run: cargo clippy --all-targets --workspace -- -D warnings + + check_fmt: + runs-on: ubuntu-24.04 + name: Checking fmt + steps: + - uses: actions/checkout@v4 + - name: Run cargo fmt + run: cargo fmt --all -- --check + + test_release: + runs-on: ubuntu-24.04 + name: Test in release mode + strategy: + matrix: + cargo-args: ['', '--features fixed-rows-to-discard'] + env: + # Run all tests with multicore-SDR enabled. + FIL_PROOFS_USE_MULTICORE_SDR: true + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev + + - name: Download the proof params + uses: ./.github/actions/proof-params-download + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run usual tests in release profile + run: cargo test --verbose --release --workspace --all-targets ${{ matrix.cargo-args }} -- --nocapture + - name: Run isolated PoRep tests in release profile + # Getting the cores does not work on GitHub Actions, hence skip that + # specific test. + run: cargo test --release -p storage-proofs-porep --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture --skip stacked::vanilla::cores::tests::test_checkout_cores + - name: Run isolated update tests in release profile + # Some `storage-proofs-update` tests need to run sequentially due to + # their high memory usage. + run: cargo test --release -p storage-proofs-update --features isolated-testing ${{ matrix.cargo-args }} -- --nocapture --test-threads=1 + + test_ignored_release: + runs-on: ubuntu-24.04 + name: Test ignored in release mode + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev + + - name: Download the proof params + uses: ./.github/actions/proof-params-download + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Test ignored in release profile + run: cargo test --release --workspace -- ignored --nocapture + + test_no_default_features: + runs-on: ubuntu-24.04 + name: Test without default features + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: sudo apt install --no-install-recommends --yes libhwloc-dev ocl-icd-opencl-dev + + - name: Download the proof params + uses: ./.github/actions/proof-params-download + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Test ignored in release profile + run: cargo test --release --workspace --no-default-features + + build_gpu: + runs-on: ubuntu-24.04 + name: Build with various GPU support enabled + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: sudo apt install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev + + - name: Build with `cuda` and `opencl` features enabled + run: cargo build --workspace --features cuda,opencl + - name: Build with `cuda-supraseal` feature enabled + run: CC=gcc-12 CXX=g++-12 NVCC_PREPEND_FLAGS='-ccbin /usr/bin/g++-12' cargo build --workspace --no-default-features --features cuda-supraseal + + # Commented out until we run it on hardware with actual GPUs. + #test_gpu: + # runs-on: ubuntu-24.04 + # name: Test on GPUs + # strategy: + # matrix: + # test-args: ['', '--ignored'] + # env: + # FIL_PROOFS_USE_GPU_COLUMN_BUILDER: true + # FIL_PROOFS_USE_GPU_TREE_BUILDER: true + # BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + # NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + # steps: + # - uses: actions/checkout@v4 + # - name: Install required packages + # run: sudo apt install --no-install-recommends --yes libhwloc-dev nvidia-cuda-toolkit ocl-icd-opencl-dev + # + # - name: Download the proof params + # uses: ./.github/actions/proof-params-download + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # + # - name: Test with CUDA + # run: cargo test --verbose --release --workspace --features cuda -- --nocapture ${{ matrix.test-args }} + # - name: Test with `cuda-supraseal` + # run: CC=gcc-12 CXX=g++-12 NVCC_PREPEND_FLAGS='-ccbin /usr/bin/g++-12' cargo test -p filecoin-proofs --release --no-default-features --features cuda-supraseal -- --nocapture --test-threads=1 ${{ matrix.test-args }} + + test_macos: + runs-on: macos-latest + name: Test in release mode on MacOS + steps: + - uses: actions/checkout@v4 + - name: Install required packages + run: HOMEBREW_NO_AUTO_UPDATE=1 brew install hwloc + + - name: Download the proof params + uses: ./.github/actions/proof-params-download + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Run usual tests in release profile + run: cargo test --verbose --release --workspace -- --nocapture diff --git a/.github/workflows/proof-params-generate.yml b/.github/workflows/proof-params-generate.yml new file mode 100644 index 000000000..7633f5902 --- /dev/null +++ b/.github/workflows/proof-params-generate.yml @@ -0,0 +1,37 @@ +name: Proof params generate + +on: + # Make it also possible to trigger it manually. + workflow_dispatch: + schedule: + # The artifacts are available for 90 days. Run this workflow every two + # months, so that in case of a failure, there's enough time to fix it. + - cron: "0 0 1 */2 *" + +env: + # Faster crates.io index checkout. + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + +jobs: + proof-params: + runs-on: ubuntu-latest + name: Generate proof parameters and cache them + strategy: + # Run the parameter generation in separate jobs, so that they can run in + # parallel. + matrix: + size: [2048, 4096, 16384, 32768] + + steps: + - uses: actions/checkout@v4 + + - name: Generate proof parameters + run: cargo run --release --no-default-features --bin paramcache -- --sector-sizes=${{ matrix.size }} + + - name: Upload proof parameters as artifact + uses: actions/upload-artifact@v4 + with: + name: proof-params-v28-n-${{ hashFiles('filecoin-proofs/parameters.json') }}-${{ matrix.size }} + path: /var/tmp/filecoin-proof-parameters/ + compression-level: 0 + if-no-files-found: error