Add regression test for memory limiter

This allows us to observe performance and memory usage when enabling memory limiter for Mountpoint in different use cases. Signed-off-by: Monthon Klongklaew <monthonk@amazon.com>
awslabs · Oct 28, 2024 · 62996ef · 62996ef
1 parent e72d7ac
commit 62996ef
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 16 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -232,3 +232,69 @@ jobs:
         comment-on-alert: false
         max-items-in-chart: 20
         skip-fetch-gh-pages: true
+
+  mem-limiter-bench:
+    name: Benchmark (Memory limiter)
+    runs-on: [self-hosted, linux, x64, nvme-high-performance]
+
+    environment: ${{ inputs.environment }}
+
+    steps:
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ vars.ACTIONS_BENCH_IAM_ROLE }}
+        aws-region: ${{ vars.S3_BENCH_REGION }}
+        role-duration-seconds: 21600
+    - name: Checkout code
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ inputs.ref }}
+        submodules: true
+        persist-credentials: false
+    - name: Install operating system dependencies
+      uses: ./.github/actions/install-dependencies
+      with:
+        fuseVersion: 2
+        libunwind: true
+        fio: true
+    - name: Set up stable Rust
+      uses: dtolnay/rust-toolchain@stable
+    - name: Build
+      # TODO: remove feature flag once stable
+      run: cargo build --release --features mem_limiter
+    - name: Run Benchmark
+      run: mountpoint-s3/scripts/fs_bench.sh
+      env:
+        OPTIONAL_COMPILE_ARGS: "--features mem_limiter"
+        # Set memory usage target for Mountpoint to 2GiB
+        OPTIONAL_MOUNT_ARGS: "--max-memory-target=2048"
+    - name: Check benchmark results
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        tool: 'customBiggerIsBetter'
+        output-file-path: results/output.json
+        benchmark-data-dir-path: dev/mem_limiter_bench
+        alert-threshold: "200%"
+        fail-on-alert: false
+        # GitHub API token to make a commit comment
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        # Store the results and deploy GitHub pages automatically if the results are from main branch
+        auto-push: ${{ inputs.publish }}
+        comment-on-alert: false
+        max-items-in-chart: 20
+    - name: Check resource utilization
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        tool: 'customSmallerIsBetter'
+        output-file-path: results/peak_mem_usage.json
+        benchmark-data-dir-path: dev/mem_limiter_bench/peak_mem_usage
+        alert-threshold: "200%"
+        fail-on-alert: false
+        # GitHub API token to make a commit comment
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        # Store the results and deploy GitHub pages automatically if the results are from main branch
+        auto-push: ${{ inputs.publish }}
+        comment-on-alert: false
+        max-items-in-chart: 20
+        skip-fetch-gh-pages: true
diff --git a/mountpoint-s3/scripts/fs_bench.sh b/mountpoint-s3/scripts/fs_bench.sh
@@ -30,10 +30,21 @@ if [[ -n "${S3_JOB_NAME_FILTER}" ]]; then
   echo "Will only run fio jobs which match $S3_JOB_NAME_FILTER"
 fi
 
-optional_args=""
+optional_compile_args=""
+optional_mount_args=""
+
+if [[ -n "${OPTIONAL_COMPILE_ARGS}" ]]; then
+  echo "Will compile mountpoint with optional arguments ${OPTIONAL_COMPILE_ARGS}"
+  optional_compile_args+="${OPTIONAL_COMPILE_ARGS}"
+fi
+
+if [[ -n "${OPTIONAL_MOUNT_ARGS}" ]]; then
+  echo "Will run mountpoint with optional arguments ${OPTIONAL_MOUNT_ARGS}"
+  optional_mount_args+="${OPTIONAL_MOUNT_ARGS}"
+fi
 
 if [[ -n "${S3_ENDPOINT_URL}" ]]; then
-  optional_args+="--endpoint-url=${S3_ENDPOINT_URL}"
+  optional_mount_args+="--endpoint-url=${S3_ENDPOINT_URL}"
 fi
 
 base_dir=$(dirname "$0")
@@ -137,14 +148,14 @@ read_benchmark () {
 
     # mount file system
     set +e
-    cargo run --quiet --release -- \
+    cargo run --quiet --release ${optional_compile_args} -- \
       ${S3_BUCKET_NAME} ${mount_dir} \
       --debug \
       --allow-delete \
       --log-directory=${log_dir} \
       --prefix=${S3_BUCKET_TEST_PREFIX} \
       --part-size=16777216 \
-      ${optional_args}
+      ${optional_mount_args}
     mount_status=$?
     set -e
     if [ $mount_status -ne 0 ]; then
@@ -203,13 +214,13 @@ write_benchmark () {
     # mount file system
     mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX)
     set +e
-    cargo run --quiet --release -- \
+    cargo run --quiet --release ${optional_compile_args} -- \
       ${S3_BUCKET_NAME} ${mount_dir} \
       --debug \
       --allow-delete \
       --log-directory=${log_dir} \
       --prefix=${S3_BUCKET_TEST_PREFIX} \
-      ${optional_args}
+      ${optional_mount_args}
     mount_status=$?
     set -e
     if [ $mount_status -ne 0 ]; then

diff --git a/mountpoint-s3/scripts/fs_cache_bench.sh b/mountpoint-s3/scripts/fs_cache_bench.sh
@@ -30,10 +30,21 @@ if [[ -n "${S3_JOB_NAME_FILTER}" ]]; then
   echo "Will only run fio jobs which match $S3_JOB_NAME_FILTER"
 fi
 
-optional_args=""
+optional_compile_args=""
+optional_mount_args=""
+
+if [[ -n "${OPTIONAL_COMPILE_ARGS}" ]]; then
+  echo "Will compile mountpoint with optional arguments ${OPTIONAL_COMPILE_ARGS}"
+  optional_compile_args+="${OPTIONAL_COMPILE_ARGS}"
+fi
+
+if [[ -n "${OPTIONAL_MOUNT_ARGS}" ]]; then
+  echo "Will run mountpoint with optional arguments ${OPTIONAL_MOUNT_ARGS}"
+  optional_mount_args+="${OPTIONAL_MOUNT_ARGS}"
+fi
 
 if [[ -n "${S3_ENDPOINT_URL}" ]]; then
-  optional_args+="--endpoint-url=${S3_ENDPOINT_URL}"
+  optional_mount_args+="--endpoint-url=${S3_ENDPOINT_URL}"
 fi
 
 base_dir=$(dirname "$0")
@@ -162,15 +173,15 @@ cache_benchmark () {
 
     # mount file system
     set +e
-    cargo run --quiet --release -- \
+    cargo run --quiet --release ${optional_compile_args} -- \
       ${S3_BUCKET_NAME} ${mount_dir} \
       --debug \
       --allow-delete \
       --cache=${cache_dir} \
       --log-directory=${log_dir} \
       --prefix=${S3_BUCKET_TEST_PREFIX} \
       --part-size=16777216 \
-      ${optional_args}
+      ${optional_mount_args}
     mount_status=$?
     set -e
     if [ $mount_status -ne 0 ]; then

diff --git a/mountpoint-s3/scripts/fs_latency_bench.sh b/mountpoint-s3/scripts/fs_latency_bench.sh
@@ -25,10 +25,21 @@ if [[ -z "${S3_BUCKET_SMALL_BENCH_FILE}" ]]; then
   exit 1
 fi
 
-optional_args=""
+optional_compile_args=""
+optional_mount_args=""
+
+if [[ -n "${OPTIONAL_COMPILE_ARGS}" ]]; then
+  echo "Will compile mountpoint with optional arguments ${OPTIONAL_COMPILE_ARGS}"
+  optional_compile_args+="${OPTIONAL_COMPILE_ARGS}"
+fi
+
+if [[ -n "${OPTIONAL_MOUNT_ARGS}" ]]; then
+  echo "Will run mountpoint with optional arguments ${OPTIONAL_MOUNT_ARGS}"
+  optional_mount_args+="${OPTIONAL_MOUNT_ARGS}"
+fi
 
 if [[ -n "${S3_ENDPOINT_URL}" ]]; then
-  optional_args+="--endpoint-url=${S3_ENDPOINT_URL}"
+  optional_mount_args+="--endpoint-url=${S3_ENDPOINT_URL}"
 fi
 
 base_dir=$(dirname "$0")
@@ -59,12 +70,13 @@ do
     echo "Running ${job_name}"
 
     # mount file system
-    cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \
+    cargo run --release ${optional_compile_args} -- \
+        ${S3_BUCKET_NAME} ${mount_dir} \
         --debug \
         --allow-delete \
         --log-directory=$log_dir \
         --prefix=${S3_BUCKET_TEST_PREFIX} \
-        ${optional_args}
+        ${optional_mount_args}
     mount_status=$?
     if [ $mount_status -ne 0 ]; then
         echo "Failed to mount file system"
@@ -135,12 +147,13 @@ for job_file in "${jobs_dir}"/*.fio; do
   echo "Running ${job_name}"
 
   # mount file system
-  cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \
+  cargo run --release ${optional_compile_args} \
+    ${S3_BUCKET_NAME} ${mount_dir} \
     --debug \
     --allow-delete \
     --log-directory=$log_dir \
     --prefix=${S3_BUCKET_TEST_PREFIX} \
-    ${optional_args}
+    ${optional_mount_args}
   mount_status=$?
   if [ $mount_status -ne 0 ]; then
     echo "Failed to mount file system"