feat(ci): add native dependencies to the CI (dynamic docker) (starkware-libs#726)

* fix: ci

* fix: return back old images in the CI

* fix: add native deps to some CI jobs

* chore: update feature contracts

* fix: add LLVM deps to the Dockerfile

* fix: add LLVM to `install_build_tools.sh`

* fix: make larger runner

* fix: make ubuntu latest runner

* fix: update build tools, add env vars

* fix: update build tools, add env vars

* fix: add clean up of the native artifacts

* fix: add clean up of the native artifacts

* fix: native artifacts push CI

* fix: native artifacts push CI

* fix: native artifacts push CI

* fix: add -y arg to apt

* fix: add env to ignore interactive env

* fix: increase size of the runners

* fix: update runner version

* fix: remove native and rust dependencies

* fix: update sequencer-ci.Dockerfile

* fix: update sequencer-ci.Dockerfile

* fix: update sequencer-ci.Dockerfile

* fix: increase runner size

* fix: try to fix Dockerfile with adding musl-g++

* fix: try to fix Dockerfile with adding musl-g++ #2

* fix: try to fix Dockerfile with adding musl-g++ #2

* fix: try to fix Dockerfile with adding musl-g++ #4

* fix: try to fix Dockerfile with adding musl-g++ #4

* fix: try to fix Dockerfile

* fix: try to fix Dockerfile finally?

* fix: try to fix Dockerfile

* fix: add hard option for sym link creation

* fix: make Dockerfile dynamic

* Merge remote-tracking branch 'origin/main' into fix/native-dynamic-ci

* feat: add script to install native deps

* feat: add script to install native deps

* fix: add sudo for running the script

* fix: add sudo for running the script

* fix: add sudo for running the script

* fix: fix llvm paths

* fix: script

* fix: script

* fix: script

* fix: script

* fix: script

* fix: script

* fix: script

* fix: script

* fix: script

* fix: verify_cairo_file_dependencies ci

* fix: cairo native runtime library path

* Merge branch 'main' into fix/native-dynamic-ci

# Conflicts:
#	Cargo.lock
#	Cargo.toml

* fix: conflicts after merge

* fix: update feature contracts

* fix: update gas in secp tests

* fix: increase merge-gatekeeper timeout to 25 minutes (1500 seconds)

Author: varex83

.github/actions/setup-native-deps/action.yml

@@ -0,0 +1,28 @@
+name: "Setup Cairo Native Dependencies"
+description: "Sets up LLVM and GMP libraries"
+
+outputs:
+  cairo-native-runtime-library:
+    description: "The path to the cairo native runtime library"
+    value: ${{ steps.set-env-vars.outputs.cairo-native-runtime-library }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Make script executable
+      shell: bash
+      run: chmod +x ./scripts/setup_native_deps.sh
+
+    - name: Install Cairo Native Runtime Dependencies
+      id: set-runtime-deps
+      shell: bash
+      run: |
+        sudo ./scripts/setup_native_deps.sh
+
+    - name: Set Environment Variables
+      id: set-env-vars
+      shell: bash
+      run: |
+        echo "MLIR_SYS_180_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV &&
+        echo "LLVM_SYS_181_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV &&
+        echo "TABLEGEN_180_PREFIX=/usr/lib/llvm-18/" >> $GITHUB_ENV

.github/workflows/blockifier_ci.yml

@@ -46,19 +46,17 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo build -p blockifier
       - run: cargo test -p blockifier
 
   native-blockifier-artifacts-push:
-    runs-on: starkware-ubuntu-20-04-medium
+    runs-on: starkware-ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/install_rust
-
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v1-rust-ubuntu-20.04"
-
+        
       - name: Build native blockifier
         run: ./build_native_in_docker.sh scripts/build_native_blockifier.sh
 

.github/workflows/blockifier_compiled_cairo.yml

@@ -40,6 +40,10 @@ jobs:
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
         id: setup-pypy

.github/workflows/blockifier_post-merge.yml

@@ -21,6 +21,10 @@ jobs:
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       # Setup pypy and link to the location expected by .cargo/config.toml.
       - uses: actions/setup-python@v5
         id: setup-pypy

.github/workflows/committer_ci.yml

@@ -76,7 +76,7 @@ jobs:
 
       # List the existing benchmarks.
       - run: |
-            cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
+          cargo bench -p committer_cli -- --list | grep ': benchmark$' | sed -e "s/: benchmark$//" > benchmarks_list.txt
 
       # Benchmark the old code.
       - run: cargo bench -p committer_cli
@@ -119,6 +119,7 @@ jobs:
               owner: context.repo.owner,
               repo: context.repo.repo,
               body: fs.readFileSync('bench_new.txt', 'utf8'),
+              path: 'Commits'
             })
 
   gcs-push:

.github/workflows/main.yml

@@ -66,6 +66,11 @@ jobs:
       - uses: ./.github/actions/install_rust
         with:
           components: clippy
+
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       - uses: Swatinem/rust-cache@v2
         with:
           prefix-key: "v0-rust-ubuntu-20.04"
@@ -89,6 +94,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - uses: Noelware/setup-protoc@1.1.0
         with:
@@ -100,17 +108,23 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: cargo test -p workspace_tests
 
   run-tests:
-    runs-on: starkware-ubuntu-20-04-medium
+    runs-on: starkware-ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
         with:
           # Fetch the entire history.
           fetch-depth: 0
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/setup-protoc@1.1.0
       - uses: Swatinem/rust-cache@v2
         with:
@@ -175,6 +189,9 @@ jobs:
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo check --workspace -r --all-features
 
   merge-gatekeeper:
@@ -190,7 +207,7 @@ jobs:
         uses: upsidr/merge-gatekeeper@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -200,7 +217,7 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{github.ref}}
-          timeout: 1200
+          timeout: 1500
           interval: 30
           ignored: "code-review/reviewable"
 
@@ -216,6 +233,9 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y clang llvm libudev-dev
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}

.github/workflows/papyrus_benchmark.yaml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
   # TODO: Uncomment and run this automatically when the storage benchmark is fixed.
   # push:
-    # branches: [main]
+  # branches: [main]
 
 jobs:
   storage-benchmark:

.github/workflows/papyrus_ci.yml

@@ -2,7 +2,7 @@ name: Papyrus-CI
 
 on:
   push:
-    branches: [main]
+    branches: [ main ]
     paths:
       - '.github/workflows/papyrus_ci.yml'
       - 'Dockerfile'
@@ -31,7 +31,7 @@ on:
       - 'crates/starknet_client/**'
 
   merge_group:
-    types: [checks_requested]
+    types: [ checks_requested ]
 
 # On PR events, cancel existing CI runs on this same PR for this workflow.
 concurrency:
@@ -48,6 +48,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -67,6 +70,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -86,6 +92,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
@@ -107,6 +116,9 @@ jobs:
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: >
           cargo test -r
           --test latency_histogram
@@ -122,6 +134,9 @@ jobs:
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
       - uses: Noelware/setup-protoc@1.1.0
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
 
       - run: |
           cargo test -p papyrus_node --no-default-features
@@ -140,7 +155,9 @@ jobs:
       - uses: Noelware/setup-protoc@1.1.0
         with:
           version: ${{env.PROTOC_VERSION}}
-
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - name: Install cargo-llvm-cov
         uses: taiki-e/install-action@cargo-llvm-cov
       - uses: Swatinem/rust-cache@v2
@@ -197,6 +214,9 @@ jobs:
             target_directory:
               - 'crates/papyrus_storage/src/db/**'
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       # repeat this job 32 times. this is a random test for part of the code that may cause a corrupted database.
       - run: for run in {1..32}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done

.github/workflows/papyrus_docker-publish.yml

@@ -3,8 +3,8 @@ name: Papyrus-Docker-Publish
 on:
   workflow_dispatch:
   push:
-    branches: [main]
-    tags: ["v*.*.*"]
+    branches: [ main ]
+    tags: [ "v*.*.*" ]
     paths:
       - '.github/workflows/papyrus_docker-publish.yml'
       - 'crates/papyrus**/**'
@@ -25,7 +25,7 @@ env:
 
 jobs:
   docker-build-push:
-    runs-on: starkware-ubuntu-latest-medium
+    runs-on: starkware-ubuntu-latest-large
 
     steps:
       - name: Checkout repository

.github/workflows/papyrus_nightly-tests-call.yml

@@ -22,6 +22,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: sudo apt update; sudo apt -y install libclang-dev
         # Install libclang-dev that is not a part of the ubuntu vm in github actions.

.github/workflows/papyrus_nightly-tests.yml

@@ -44,6 +44,10 @@ jobs:
 
       - run: brew install protobuf@$PROTOC_VERSION
 
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
+
       - name: Build node
         run: cargo build -r -p papyrus_node
 
@@ -58,6 +62,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - uses: Swatinem/rust-cache@v2
       - run: npm install -g ganache@7.4.3
 
@@ -75,6 +82,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: cargo build -r -p papyrus_load_test
 
   integration-test:
@@ -84,6 +94,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: >
           cargo test -r
           --test latency_histogram
@@ -100,4 +113,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
       - uses: Swatinem/rust-cache@v2
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - run: for run in {1..100}; do cargo test -r -p papyrus_storage -- --include-ignored common_prefix_compare_with_simple_table_random; done

.github/workflows/verify-deps.yml

@@ -12,6 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install_rust
+      - name: Set up Native Dependencies
+        uses: ./.github/actions/setup-native-deps
+        id: native-deps
       - name: Update Dependencies
         run: cargo update --verbose
       - name: Build