Merge pull request #253 from matter-labs/sb-merge-main-into-boojum

chore: Merge main into boojum
matter-labs · Oct 18, 2023 · 71ffe99 · 71ffe99
2 parents 5e7bc64 + 4c2d597
commit 71ffe99
Show file tree

Hide file tree

Showing 242 changed files with 10,927 additions and 8,638 deletions.
diff --git a/.githooks/pre-commit b/.githooks/pre-commit
@@ -7,8 +7,16 @@ RED='\033[0;31m'
 NC='\033[0m' # No Color
 
 # Check that Rust formatting rules are not violated.
-if ! cargo fmt -- --check; then
-    echo -e "${RED}Commit error!${NC}"
-    echo "Please format the code via 'cargo fmt', cannot commit unformatted code"
-    exit 1
-fi
+function check_fmt {
+    if ! cargo fmt -- --check; then
+        echo -e "${RED}Commit error!${NC}"
+        echo "Please format the code via 'cargo fmt', cannot commit unformatted code"
+        exit 1
+    fi
+}
+
+check_fmt
+
+cd prover/
+
+check_fmt
diff --git a/.github/release-please/config.json b/.github/release-please/config.json
@@ -1,7 +1,7 @@
 {
   "separate-pull-requests": true,
   "group-pull-request-title-pattern": "chore: release ${component} ${branch}",
-  "bootstrap-sha": "531757b5eb98da80b7e6d0ff7ad9fd0b970cc109",
+  "bootstrap-sha": "691a7008f6d1f88fb9a5b6b8d92592e1199f37ea",
   "bump-minor-pre-major": true,
   "bump-patch-for-minor-pre-major": true,
   "packages": {

diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,6 +1,6 @@
 {
   "sdk/zksync-web3.js": "0.15.4",
   "sdk/zksync-rs": "0.4.0",
-  "core": "16.0.0",
-  "prover": "7.1.1"
+  "core": "16.0.2",
+  "prover": "7.2.0"
 }
diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml
@@ -71,6 +71,12 @@ jobs:
           ci_run zk f yarn run l2-contracts build
           ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
+      - name: login to Docker registries
+        if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        run: |
+          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q
+
       - name: update-images
         env:
           DOCKER_ACTION: ${{ inputs.action }}

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -26,6 +26,11 @@ on:
         type: string
         default: "push"
         required: false
+      is_pr_from_fork:
+        description: "Indicates whether the workflow is invoked from a PR created from fork"
+        type: boolean
+        default: false
+        required: false
 
 jobs:
   build-images:
@@ -81,6 +86,55 @@ jobs:
           ci_run zk f yarn run l2-contracts build
           ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
+      - name: login to Docker registries
+        if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        run: |
+          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q
+
+      # We need to run this only when ERA_BELLMAN_CUDA_RELEASE is not available
+      # In our case it happens only when PR is created from fork
+      - name: Wait for runner IP to be not rate-limited against GH API
+        if: inputs.is_pr_from_fork == true
+        run: |
+          api_endpoint="https://api.github.com/users/zksync-era-bot"
+          wait_time=60
+          max_retries=60
+          retry_count=0
+
+          while [[ $retry_count -lt $max_retries ]]; do
+            response=$(curl -s -w "%{http_code}" -o temp.json "$api_endpoint")
+            http_code=$(echo "$response" | tail -n1)
+
+            if [[ "$http_code" == "200" ]]; then
+              echo "Request successful. Not rate-limited."
+              cat temp.json
+              rm temp.json
+              exit 0
+            elif [[ "$http_code" == "403" ]]; then
+              rate_limit_exceeded=$(jq -r '.message' temp.json | grep -i "API rate limit exceeded")
+              if [[ -n "$rate_limit_exceeded" ]]; then
+                retry_count=$((retry_count+1))
+                echo "API rate limit exceeded. Retry $retry_count of $max_retries. Retrying in $wait_time seconds..."
+                sleep $wait_time
+              else
+                echo "Request failed with HTTP status $http_code."
+                cat temp.json
+                rm temp.json
+                exit 1
+              fi
+            else
+              echo "Request failed with HTTP status $http_code."
+              cat temp.json
+              rm temp.json
+              exit 1
+            fi
+          done
+
+          echo "Reached the maximum number of retries ($max_retries). Exiting."
+          rm temp.json
+          exit 1
+
       - name: update-images
         env:
           DOCKER_ACTION: ${{ inputs.action }}
@@ -98,8 +152,6 @@ jobs:
             )
         run: |
           nightly_components=${{ env.RUST_NIGHTLY_COMPONENTS }}
-          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
-          ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q
 
           if [[ " ${nightly_components[*]} " =~ " ${COMPONENT} " ]]; then
             underscored_name=$(echo "$COMPONENT" | tr '-' '_')

diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml
@@ -0,0 +1,40 @@
+name: Workflow template for Core Linting CI jobs
+on:
+  workflow_call:
+
+
+jobs:
+  code_lint:
+    runs-on: [matterlabs-ci-runner]
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          docker-compose -f docker-compose-runner.yml pull
+          docker-compose -f docker-compose-runner.yml up --build -d zk
+          ci_run sccache --start-server
+
+      - name: Setup db
+        run: |
+          ci_run zk
+          ci_run zk db migrate
+
+      - name: Lints
+        run: |
+          ci_run zk fmt --check
+          ci_run zk lint rust --check
+          ci_run zk lint js --check
+          ci_run zk lint ts --check
+          ci_run zk lint md --check
+          ci_run zk db check-sqlx-data
+
diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -4,39 +4,8 @@ on:
 
 jobs:
   lint:
-    runs-on: [matterlabs-ci-runner]
-
-    steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
-        with:
-          submodules: "recursive"
-
-      - name: Setup environment
-        run: |
-          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
-          echo $(pwd)/bin >> $GITHUB_PATH
-          echo IN_DOCKER=1 >> .env
-
-      - name: Start services
-        run: |
-          docker-compose -f docker-compose-runner.yml pull
-          docker-compose -f docker-compose-runner.yml up --build -d zk
-          ci_run sccache --start-server
-
-      - name: Setup db
-        run: |
-          ci_run zk
-          ci_run zk db migrate
-
-      - name: Lints
-        run: |
-          ci_run zk fmt --check
-          ci_run zk lint rust --check
-          ci_run zk lint js --check
-          ci_run zk lint ts --check
-          ci_run zk lint md --check
-          ci_run zk db check-sqlx-data
-
+    name: lint
+    uses: ./.github/workflows/ci-core-lint-reusable.yml
   unit-tests:
     runs-on: [matterlabs-ci-runner]
 

diff --git a/.github/workflows/ci-docs-reusable.yml b/.github/workflows/ci-docs-reusable.yml
@@ -0,0 +1,30 @@
+name: Workflow template for CI jobs against docs
+on:
+  workflow_call:
+
+jobs:
+  lint:
+    runs-on: [matterlabs-ci-runner]
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          docker-compose -f docker-compose-runner.yml pull
+          docker-compose -f docker-compose-runner.yml up --build -d zk
+
+      - name: Lints
+        run: |
+          ci_run zk
+          ci_run zk fmt md --check
+          ci_run zk lint md --check
+
diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml
@@ -3,6 +3,39 @@ on:
   workflow_call:
 
 jobs:
+  lint:
+    runs-on: [matterlabs-ci-runner]
+    env:
+      RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
+
+    steps:
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+        with:
+          submodules: "recursive"
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          docker-compose -f ${RUNNER_COMPOSE_FILE} pull
+          docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk
+          ci_run sccache --start-server
+
+      - name: Init
+        run: |
+          ci_run zk
+          ci_run zk db setup
+
+      - name: Formatting
+        run: ci_run bash -c "cd prover && cargo fmt --check"
+
+      - name: Lints
+        run: ci_run zk lint prover
+
   unit-tests:
     runs-on: [matterlabs-ci-runner]
     env:
@@ -22,16 +55,15 @@ jobs:
       - name: Start services
         run: |
           docker-compose -f ${RUNNER_COMPOSE_FILE} pull
-          docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d geth zk postgres
+          docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d zk
           ci_run sccache --start-server
 
       - name: Init
         run: |
           ci_run zk
-          ci_run zk config compile
           ci_run zk db setup
 
       - name: Prover unit tests
         run: |
-          # Not all tests are enabled since, prover and setup_key_generator_and_server requires bellman-cuda to be present
-          ci_run bash -c "cd prover && zk f cargo test --release -p zksync_witness_generator -p vk_setup_data_generator_server_fri -p zksync_prover_fri -p zksync_witness_vector_generator -p zksync_prover_fri_utils"
+          # Not all tests are enabled, since prover and setup_key_generator_and_server requires bellman-cuda to be present
+          ci_run zk test prover