Add sequential write benchmark

.github/workflows/bench.yml

@@ -62,7 +62,7 @@ jobs:
           target/
         key: ${{ runner.os }}-${{ github.job }}-cargo-${{ hashFiles('**/Cargo.lock') }}
     - name: Build
-      run: cargo build --release
+      run: cargo build --release --features delete
     - name: Run Benchmark
       run: mountpoint-s3/scripts/fs_bench.sh
     - name: Save Cargo cache
@@ -131,7 +131,7 @@ jobs:
           target/
         key: ${{ runner.os }}-${{ github.job }}-cargo-${{ hashFiles('**/Cargo.lock') }}
     - name: Build
-      run: cargo build --release
+      run: cargo build --release --features delete
     - name: Run Benchmark
       run: mountpoint-s3/scripts/fs_latency_bench.sh
     - name: Save Cargo cache

doc/BENCHMARKING.md

@@ -13,6 +13,8 @@ In general, we run each IO operation for 30 seconds against a 100 GiB file. But
 
 ***readdir workload*** - we measure how long it takes to run `ls` command against directories with different size. Each directory has no subdirectory and contains a specific number of files, range from 100 to 100000 files, which we have to create manually using fio then upload them to S3 bucket before running the benchmark. The fio configuration files for creating them can be found at path [mountpoint-s3/scripts/fio/create/](../mountpoint-s3/scripts/fio/create).
 
+***write workload*** - we measure write throughput by using [dd](https://man7.org/linux/man-pages/man1/dd.1.html) command to simulate sequential write workloads. We plan to use fio in the future for consistency with other benchmarks but its current write pattern is not supported by Mountpoint. Firstly, fio creates a file with 0 byte and close it. Secondly, fio opens the file again with `O_RDWR` flag to do the IO workloads. To support fio, Mountpoint has to allow file overwrites and allow file opens with `O_RDWR` flag.
+
 ### Regression Testing
 Our CI runs the benchmark automatically for any new commits to the main branch or specific pull requests that we have reviewed and tagged with **performance** label. Every benchmark from the CI workflow will be running on `m5n.24xlarge` EC2 instances (100 Gbps network speed) with Ubuntu 22.04 in us-east-1 against a bucket in us-east-1.
 

mountpoint-s3/scripts/fs_bench.sh

@@ -30,77 +30,159 @@ project_dir="${base_dir}/../.."
 cd ${project_dir}
 
 results_dir=results
-jobs_dir=mountpoint-s3/scripts/fio/read
+runtime_seconds=30
+startdelay_seconds=30
 thread_count=4
 
 rm -rf ${results_dir}
 mkdir -p ${results_dir}
 
-for job_file in "${jobs_dir}"/*.fio; do
-  mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX)
-  job_name=$(basename "${job_file}")
-  job_name="${job_name%.*}"
+read_bechmark () {
+  jobs_dir=mountpoint-s3/scripts/fio/read
 
-  echo "Running ${job_name}"
-
-  # mount file system
-  cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \
-    --foreground \
-    --prefix=${S3_BUCKET_TEST_PREFIX} \
-    --thread-count=${thread_count} > bench.out 2>&1 &
-  # get file system PID
-  fs_pid=$!
-
-  start_time="$(date -u +%s)"
-  timeout_seconds=30
-  # wait for file system to be ready
-  while true; do
-    mount_rec=`findmnt -rncv -S mountpoint-s3 -T ${mount_dir} -o SOURCE,TARGET`
-    # file system is ready when the mount record exists
-    if [ -n "${mount_rec}" ]; then
-      break
-    fi
-    sleep 0.01
+  for job_file in "${jobs_dir}"/*.fio; do
+    mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX)
+    job_name=$(basename "${job_file}")
+    job_name="${job_name%.*}"
+
+    echo "Running ${job_name}"
 
-    # exit and fail when file system is not ready before timeout
-    end_time="$(date -u +%s)"
-    elapsed="$(($end_time-$start_time))"
-    if [ "$elapsed" -gt "$timeout_seconds" ]; then
-      echo "Timeout while waiting for file system to be ready"
+    # mount file system
+    cargo run --release --features delete ${S3_BUCKET_NAME} ${mount_dir} \
+      --prefix=${S3_BUCKET_TEST_PREFIX} \
+      --thread-count=${thread_count}
+    mount_status=$?
+    if [ $mount_status -ne 0 ]; then
+      echo "Failed to mount file system"
       exit 1
     fi
+
+    # set bench file
+    bench_file=${S3_BUCKET_BENCH_FILE}
+    # run against small file if the job file ends with small.fio
+    if [[ $job_file == *small.fio ]]; then
+      bench_file=${S3_BUCKET_SMALL_BENCH_FILE}
+    fi
+
+    # run benchmark
+    fio --thread \
+      --output=${results_dir}/${job_name}.json \
+      --output-format=json \
+      --directory=${mount_dir} \
+      --filename=${bench_file} \
+      ${job_file}
+
+    # unmount file system
+    sudo umount ${mount_dir}
+
+    # cleanup mount directory
+    rm -rf ${mount_dir}
+
+    # parse result
+    jq -n 'inputs.jobs[] | if (."job options".rw == "read")
+      then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"}
+      elif (."job options".rw == "randread") then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"}
+      elif (."job options".rw == "randwrite") then {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"}
+      else {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"} end' ${results_dir}/${job_name}.json | tee ${results_dir}/${job_name}_parsed.json
+
+    # delete the raw output file
+    rm ${results_dir}/${job_name}.json
   done
+}
 
-  # set bench file
-  bench_file=${S3_BUCKET_BENCH_FILE}
-  # run against small file if the job file ends with small.fio
-  if [[ $job_file == *small.fio ]]; then
-    bench_file=${S3_BUCKET_SMALL_BENCH_FILE}
+write_benchmark () {
+  # mount file system
+  mount_dir=$(mktemp -d /tmp/fio-XXXXXXXXXXXX)
+  cargo run --release --features delete ${S3_BUCKET_NAME} ${mount_dir} \
+      --prefix=${S3_BUCKET_TEST_PREFIX} \
+      --thread-count=${thread_count}
+  mount_status=$?
+  if [ $mount_status -ne 0 ]; then
+      echo "Failed to mount file system"
+      exit 1
   fi
-
-  # run benchmark
-  fio --thread \
-    --output=${results_dir}/${job_name}.json \
-    --output-format=json \
-    --directory=${mount_dir} \
-    --filename=${bench_file} \
-    ${job_file}
+  sleep $startdelay_seconds
+
+  ## sequential write
+  job_name="sequential_write"
+  bench_file=${mount_dir}/${job_name}_${RANDOM}.dat
+  dd if=/dev/zero of=$bench_file bs=256k conv=fsync > ${results_dir}/${job_name}.txt 2>&1 &
+  # get the process ID
+  dd_pid=$!
+
+  sleep $runtime_seconds
+  # send USR1 signal to print the result
+  kill -USR1 ${dd_pid}
+  sleep 0.1
+  kill ${dd_pid}
+
+  throughput_value=$(awk '/copied/ {print $10}' ${results_dir}/${job_name}.txt)
+  unit=$(awk '/copied/ {print $11}' ${results_dir}/${job_name}.txt)
+  # convert unit to MiB/s
+  case "$unit" in
+    GB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000*1000/1024/1024}")
+      ;;
+    MB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000/1024/1024}")
+      ;;
+    kB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000/1024/1024}")
+      ;;
+  esac
+
+  json_data="{\"name\":\"$job_name\",\"value\":$throughput_value,\"unit\":\"MiB/s\"}"
+  echo $json_data | jq '.' | tee ${results_dir}/${job_name}.json
+
+  # clean up the data file and the raw output file
+  sleep 10
+  rm $bench_file ${results_dir}/${job_name}.txt
+
+
+  ## sequential write with direct IO
+  job_name="sequential_write_direct_io"
+  bench_file=${mount_dir}/${job_name}_${RANDOM}.dat
+  dd if=/dev/zero of=$bench_file bs=256k oflag=direct conv=fsync > ${results_dir}/${job_name}.txt 2>&1 &
+  # get the process ID
+  dd_pid=$!
+
+  sleep $runtime_seconds
+  # send USR1 signal to print the result
+  kill -USR1 ${dd_pid}
+  sleep 0.1
+  kill ${dd_pid}
+
+  throughput_value=$(awk '/copied/ {print $10}' ${results_dir}/${job_name}.txt)
+  unit=$(awk '/copied/ {print $11}' ${results_dir}/${job_name}.txt)
+  # convert unit to MiB/s
+  case "$unit" in
+    GB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000*1000/1024/1024}")
+      ;;
+    MB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000*1000/1024/1024}")
+      ;;
+    kB/s)
+      throughput_value=$(awk "BEGIN {print $throughput_value*1000/1024/1024}")
+      ;;
+  esac
+
+  json_data="{\"name\":\"$job_name\",\"value\":$throughput_value,\"unit\":\"MiB/s\"}"
+  echo $json_data | jq '.' | tee ${results_dir}/${job_name}.json
+
+  # clean up the data file and the raw output file
+  sleep 10
+  rm $bench_file ${results_dir}/${job_name}.txt
 
   # unmount file system
   sudo umount ${mount_dir}
 
-  # kill the file system process if it's still running
-  if ps -p ${fs_pid} > /dev/null 2>&1; then
-    kill -9 ${fs_pid}
-  fi
-
   # cleanup mount directory
   rm -rf ${mount_dir}
-done
-
-# parse result
-jq -n '[inputs.jobs[] | if (."job options".rw == "read") 
-  then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"} 
-  elif (."job options".rw == "randread") then {name: .jobname, value: (.read.bw / 1024), unit: "MiB/s"} 
-  elif (."job options".rw == "randwrite") then {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"} 
-  else {name: .jobname, value: (.write.bw / 1024), unit: "MiB/s"} end]' ${results_dir}/*.json | tee ${results_dir}/output.json
+}
+
+read_bechmark
+write_benchmark
+
+# combine all bench results into one json file
+jq -n '[inputs]' ${results_dir}/*.json | tee ${results_dir}/output.json

mountpoint-s3/scripts/fs_latency_bench.sh

@@ -48,7 +48,7 @@ do
     echo "Running ${job_name}"
 
     # mount file system
-    cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \
+    cargo run --release --features delete ${S3_BUCKET_NAME} ${mount_dir} \
         --prefix=${S3_BUCKET_TEST_PREFIX} \
         --thread-count=${thread_count}
     mount_status=$?
@@ -109,7 +109,7 @@ for job_file in "${jobs_dir}"/*.fio; do
   echo "Running ${job_name}"
 
   # mount file system
-  cargo run --release ${S3_BUCKET_NAME} ${mount_dir} \
+  cargo run --release --features delete ${S3_BUCKET_NAME} ${mount_dir} \
     --prefix=${S3_BUCKET_TEST_PREFIX} \
     --thread-count=${thread_count}
   mount_status=$?