Simulator Test workflow improvements (MystenLabs#8722)

## Description 
Simulator Test workflow improvements

## Test Plan

---------

Co-authored-by: Mark Logan <mark@mystenlabs.com>

.config/nextest.toml

@@ -24,5 +24,16 @@ retries = 2
 # Mark tests as slow after 25mins, kill them after 50
 slow-timeout = { period = "1500s", terminate-after = 1 }
 
+[profile.simtestnightly]
+# Print out output for failing tests as soon as they fail, and also at the end
+# of the run (for easy scrollability).
+failure-output = "immediate-final"
+# Show skipped tests in the CI output.
+status-level = "skip"
+# Do not cancel the test run on the first failure.
+fail-fast = false
+# Mark tests as slow after 20m, terminate after 1h
+slow-timeout = { period = "20m", terminate-after = 3 }
+
 [profile.ci.junit]
 path = "junit.xml"

.github/workflows/simulator-nightly.yml

@@ -1,4 +1,4 @@
-name: simtest-nightly
+name: Simulator Tests
 
 on:
   schedule:
@@ -31,50 +31,61 @@ jobs:
   simtest:
     timeout-minutes: 90
     runs-on: [ubuntu-ghcloud]
+
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-      - uses: taiki-e/install-action@nextest
-      - name: cargo simtest
-        run: |
-          export MSIM_TEST_SEED=$(date +%s)
-          echo "Using MSIM_TEST_SEED=$MSIM_TEST_SEED"
-          MSIM_TEST_NUM=20 MSIM_WATCHDOG_TIMEOUT_MS=60000 scripts/simtest/cargo-simtest simtest
+    - name: Checkout sui repo main branch
+      uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # pin@v3
+
+    - name: Install rust toolchain
+      uses: actions-rs/toolchain@v1
+
+    - name: Install nexttest
+      uses: taiki-e/install-action@nextest
+
+    - name: Build simtest
+      run: |
+        export RUST_LOG=off
+        ./scripts/simtest/cargo-simtest simtest build
+
+    - name: Run simtest
+      run: |
+        export RUST_LOG=off
+        ./scripts/simtest/simtest-run.sh
+
+  notify:
+    name: Notify
+    needs: [simtest]
+    runs-on: [self-hosted, self-hosted-arc]
+    if: always() # always notify
 
-  report-status:
-    name: Report Status
-    runs-on: ubuntu-latest
-    if: always()
-    needs: [release]
     steps:
-      - uses: technote-space/workflow-conclusion-action@v3
-      - name: Post to slack
-        uses: slackapi/slack-github-action@v1.23.0
-        env:
-          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
-        if: env.WORKFLOW_CONCLUSION == 'failure' # notify only if failure
-        with:
-          channel-id: 'simtest-nightly'
-          payload: |
-            {
-              "text": "Workflow *${{ github.workflow }}* failed",
-              "blocks": [
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "Workflow *${{ github.workflow }}* failed"
-                  }
-                },
-                {
-                  "type": "divider"
-                },
-                {
-                  "type": "section",
-                  "text": {
-                    "type": "mrkdwn",
-                    "text": "Logs are here: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-                  }
+    - uses: technote-space/workflow-conclusion-action@v3
+    - name: Post to slack
+      uses: slackapi/slack-github-action@936158bbe252e9a6062e793ea4609642c966e302 # pin@v1.21.0
+      env:
+        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
+      with:
+        channel-id: 'simtest-nightly'
+        payload: |
+          {
+            "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`",
+            "blocks": [
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "*${{ github.workflow }}* workflow status: `${{ env.WORKFLOW_CONCLUSION }}`"
+                }
+              },
+              {
+                "type": "divider"
+              },
+              {
+                "type": "section",
+                "text": {
+                  "type": "mrkdwn",
+                  "text": "Logs are here: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                 }
-              ]
-            }
+              }
+            ]
+          }

scripts/simtest/cargo-simtest

@@ -22,11 +22,11 @@ while true; do
   MANIFEST_DIR=$(pwd)
 done
 
-cd "$MANIFEST_DIR"
-if ! git diff --quiet Cargo.lock; then
-  echo "Please commit or revert your changes to Cargo.lock before running cargo simtest"
-  exit 1
-fi
+# cd "$MANIFEST_DIR"
+# if ! git diff --quiet Cargo.lock; then
+#   echo "Please commit or revert your changes to Cargo.lock before running cargo simtest"
+#   exit 1
+# fi
 cd "$STARTING_DIR"
 
 cleanup () {

scripts/simtest/simtest-run.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Copyright (c) Mysten Labs, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+NUM_CPUS=$(cat /proc/cpuinfo | grep processor | wc -l) # ubuntu
+# NUM_CPUS=$(sysctl -n hw.ncpu) # mac
+
+DATE=$(date +%s)
+
+# TODO: increase to 30, 1000 respectively after workflow is debugged.
+export MSIM_TEST_NUM=30
+export SIM_STRESS_TEST_DURATION_SECS=1000
+
+SEED="$DATE"
+# LOG_FILE="log-$SEED"
+
+# This command runs many different tests, so it already uses all CPUs fairly efficiently, and
+# don't need to be done inside of the for loop below.
+# TODO: this logs directly to stdout since it is not being run in parallel. is that ok?
+MSIM_TEST_SEED="$SEED" MSIM_WATCHDOG_TIMEOUT_MS=60000 scripts/simtest/cargo-simtest simtest --package sui --package sui-core --profile simtestnightly
+
+for SUB_SEED in `seq 1 $NUM_CPUS`; do
+  SEED="$SUB_SEED$DATE"
+  #LOG_FILE="log-$SEED"
+  echo "Iteration $SUB_SEED using MSIM_TEST_SEED=$SEED" # logging to $LOG_FILE"
+
+  # TODO: currently this only runs one test, (`test_simulated_load_basic`).
+  # we need to add `--run-ignored all` in order to enable the ignored tests.
+  # However, the ignored tests are totally broken right now because of
+  # https://github.com/MystenLabs/sui/pull/8244.
+  #
+  # Note that because of --no-capture, even though we are running many tests, they will be
+  # serialized here. So we still need the for loop / backgrounding.
+  MSIM_TEST_SEED="$SEED" MSIM_TEST_NUM=1 MSIM_WATCHDOG_TIMEOUT_MS=60000 scripts/simtest/cargo-simtest simtest --package sui-benchmark --test-threads 1 --profile simtestnightly & # > "$LOG_FILE" 2>&1 &
+done
+
+# wait for all the jobs to end
+wait