Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adding Parallelism via ParallelTransferOptions.maxConcurrency to Stress Test Scenarios #42336

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open

Adding Parallelism via ParallelTransferOptions.maxConcurrency to Stress Test Scenarios #42336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
30ed5ae
updating downloadToFile to use maxConcurrency
ibrahimrabab Oct 12, 2024
3b1961b
added ParallelTransferOptions to applicable scenarios
ibrahimrabab Oct 14, 2024
6dc7798
updating parallelTransferOptions, updating parallelRuns
ibrahimrabab Oct 18, 2024
c0b66c3
updating datalake and file shares
ibrahimrabab Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 50 additions & 50 deletions sdk/storage/azure-storage-blob-stress/scenarios-matrix.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ matrix:
testScenario: downloadtofile
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -15,7 +15,7 @@ matrix:
testScenario: downloadtofile
sync: false
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -25,7 +25,7 @@ matrix:
testScenario: downloadtofile
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
downloadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -35,7 +35,7 @@ matrix:
testScenario: downloadstream
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -45,7 +45,7 @@ matrix:
testScenario: downloadstream
sync: false
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -55,7 +55,7 @@ matrix:
testScenario: downloadstream
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
downloadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -65,7 +65,7 @@ matrix:
testScenario: downloadcontent
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -86,7 +86,7 @@ matrix:
testScenario: downloadcontent
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
downloadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -96,7 +96,7 @@ matrix:
testScenario: openinputstream
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -106,7 +106,7 @@ matrix:
testScenario: openinputstream
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
downloadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -116,7 +116,7 @@ matrix:
testScenario: openseekablebytechannelread
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
downloadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -126,7 +126,7 @@ matrix:
testScenario: openseekablebytechannelread
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
downloadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -136,7 +136,7 @@ matrix:
testScenario: appendblock
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -146,7 +146,7 @@ matrix:
testScenario: appendblock
sync: false
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -156,7 +156,7 @@ matrix:
testScenario: appendblock
sync: true
sizeBytes: "26214400"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand Down Expand Up @@ -186,7 +186,7 @@ matrix:
testScenario: appendbloboutputstream
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -196,7 +196,7 @@ matrix:
testScenario: appendbloboutputstream
sync: true
sizeBytes: "10240"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -206,7 +206,7 @@ matrix:
testScenario: blockblobupload
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -216,7 +216,7 @@ matrix:
testScenario: blockblobupload
sync: true
sizeBytes: "26214400"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand Down Expand Up @@ -246,7 +246,7 @@ matrix:
testScenario: blockbloboutputstream
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -256,7 +256,7 @@ matrix:
testScenario: blockbloboutputstream
sync: true
sizeBytes: "26214400"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -266,7 +266,7 @@ matrix:
testScenario: commitblocklist
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -276,17 +276,17 @@ matrix:
testScenario: commitblocklist
sync: true
sizeBytes: "26214400"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 25
durationMin: 30
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to block blob open seekable byte channel write, no chunking
bytechannelwritesmall:
testScenario: openseekablebytechannelwrite
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
imageBuildDir: "../../.."
Expand All @@ -296,29 +296,29 @@ matrix:
testScenario: openseekablebytechannelwrite
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 25
durationMin: 30
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to block blob stage block, no chunking
stageblocksmall:
testScenario: stageblock
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 25MB (1024 * 1024 * 25) to block blob stage block, requiring multiple requests
stageblocklarge:
testScenario: stageblock
sync: true
sizeBytes: "26214400"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 25
durationMin: 30
imageBuildDir: "../../.."

# # this test uploads 1KB (1024 bytes) to block blob stage block from url, no chunking
Expand Down Expand Up @@ -346,87 +346,87 @@ matrix:
testScenario: pagebloboutputstream
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 10KB (1024 * 10) to page blob output stream, requiring multiple requests
pageoutputstreamlarge:
testScenario: pagebloboutputstream
sync: true
sizeBytes: "10240"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 25
durationMin: 30
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to page blob upload pages, no chunking
uploadpagessmall:
testScenario: uploadpages
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 4MB (1024 * 1024 * 4) to page blob upload pages, requiring multiple requests
uploadpageslarge:
testScenario: uploadpages
sync: true
sizeBytes: "4194304"
parallelRuns: 25
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 30
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to page blob upload pages from url, no chunking
uploadpagesfromurlsmall:
testScenario: uploadpagesfromurl
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 4MB (1024 * 1024 * 4) to page blob upload pages from url, requiring multiple requests
uploadpagesfromurllarge:
testScenario: uploadpagesfromurl
sync: true
sizeBytes: "4194304"
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 30
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to blob, no chunking
uploadsmall:
testScenario: upload
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 1KB (1024 bytes) to blob, no chunking
uploadasyncsmall:
testScenario: upload
sync: false
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 10
durationMin: 25
imageBuildDir: "../../.."

# this test uploads 4MB (1024 * 1024 * 4) to blob, requiring multiple requests
uploadlarge:
testScenario: upload
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
Expand All @@ -436,7 +436,7 @@ matrix:
testScenario: uploadfromfile
sync: true
sizeBytes: 1024
parallelRuns: 20
parallelRuns: 5
uploadFaults: true
durationMin: 25
imageBuildDir: "../../.."
Expand All @@ -446,7 +446,7 @@ matrix:
testScenario: uploadfromfile
sync: true
sizeBytes: "52428800"
parallelRuns: 10
parallelRuns: 5
uploadFaults: true
durationMin: 30
imageBuildDir: "../../.."
2 changes: 1 addition & 1 deletion ...ure-storage-blob-stress/src/main/java/com/azure/storage/blob/stress/BlobScenarioBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@
import com.azure.storage.blob.BlobServiceAsyncClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
import com.azure.storage.common.ParallelTransferOptions;
import com.azure.storage.stress.TelemetryHelper;
import com.azure.storage.stress.FaultInjectionProbabilities;
import com.azure.storage.stress.FaultInjectingHttpPolicy;
import com.azure.storage.stress.StorageStressOptions;
import reactor.core.publisher.Mono;

import java.time.Instant;
import java.util.Objects;
import java.util.UUID;

public abstract class BlobScenarioBase<TOptions extends StorageStressOptions> extends PerfStressTest<TOptions> {
Expand Down
Loading