Skip to content

Commit

Permalink
Added support for test-procedure, exclude-tasks and include-tasks ben…
Browse files Browse the repository at this point in the history 
…chmark arguments (#4180)

Signed-off-by: Rishabh Singh <sngri@amazon.com>
  • Loading branch information
@rishabh6788
rishabh6788 authored Oct 30, 2023
1 parent aa05541 commit 44a68e4
Show file tree
Hide file tree
Showing 7 changed files with 62 additions and 32 deletions.
25 changes: 23 additions & 2 deletions jenkins/opensearch/benchmark-test.jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
* compatible open source license.
*/

lib = library(identifier: 'jenkins@5.10.2', retriever: modernSCM([
lib = library(identifier: 'jenkins@5.11.0', retriever: modernSCM([

$class: 'GitSCMSource',
remote: 'https://github.com/opensearch-project/opensearch-build-libraries.git',
Expand Down Expand Up @@ -88,7 +88,7 @@ pipeline {
H 16 * * * %DISTRIBUTION_URL=https://d2xpwn7ejvoh5u.cloudfront.net/benchmark-elastic-tarball/elasticsearch-7.10.2-single-node.tar.gz;DISTRIBUTION_VERSION=7.10.2;TEST_WORKLOAD=nyc_taxis;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:1x,cluster-config:x64-r5.xlarge-single-node-es-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
H 16 * * * %DISTRIBUTION_URL=https://d2xpwn7ejvoh5u.cloudfront.net/benchmark-elastic-tarball/elasticsearch-7.10.2-single-node.tar.gz;DISTRIBUTION_VERSION=7.10.2;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:1x,cluster-config:x64-r5.xlarge-single-node-es-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
H 16 * * * %DISTRIBUTION_URL=https://artifacts.opensearch.org/releases/core/opensearch/2.11.0/opensearch-min-2.11.0-linux-x64.tar.gz;DISTRIBUTION_VERSION=2.11.0;TEST_WORKLOAD=pmc;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}

H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.12.0-SNAPSHOT/opensearch-min-2.12.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.12.0-SNAPSHOT/opensearch-min-2.12.0-SNAPSHOT-linux-x64-latest.tar.gz.build-manifest.yml;TEST_WORKLOAD=http_logs;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:x64,instance-type:r5.xlarge,major-version:2x,cluster-config:x64-r5.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
H 17 * * * %BUNDLE_MANIFEST_URL=https://artifacts.opensearch.org/snapshots/core/opensearch/2.12.0-SNAPSHOT/opensearch-min-2.12.0-SNAPSHOT-linux-arm64-latest.tar.gz.build-manifest.yml;SINGLE_NODE_CLUSTER=true;USE_50_PERCENT_HEAP=true;MIN_DISTRIBUTION=true;CAPTURE_NODE_STAT=true;USER_TAGS=run-type:nightly,segrep:disabled,arch:arm64,instance-type:r6g.xlarge,major-version:2x,cluster-config:arm64-r6g.xlarge-single-node-min-distribution;WORKLOAD_PARAMS={"number_of_replicas":"0","number_of_shards":"1"}
Expand Down Expand Up @@ -207,6 +207,21 @@ pipeline {
description: 'With this parameter you can inject variables into workloads. Use json type. e.g., `{"number_of_replicas":"1","number_of_shards":"5"}`',
trim: true
)
string(
name: 'TEST_PROCEDURE',
description: 'Defines a test procedure to use. e.g., `append-no-conflicts,significant-text`',
trim: true
)
string(
name: 'EXCLUDE_TASKS',
description: 'Defines a comma-separated list of test procedure tasks not to run. Default runs all. e.g., `type:search,delete-index`',
trim: true
)
string(
name: 'INCLUDE_TASKS',
description: 'Defines a comma-separated list of test procedure tasks to run. Default runs all. e.g., `type:search,delete-index`',
trim: true
)
booleanParam(
name: 'CAPTURE_NODE_STAT',
description: 'Enable opensearch-benchmark node-stats telemetry to capture system level metrics.',
Expand Down Expand Up @@ -304,6 +319,9 @@ pipeline {
dataInstanceType: DATA_INSTANCE_TYPE,
userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true",
workloadParams: WORKLOAD_PARAMS,
testProcedure: TEST_PROCEDURE,
excludeTasks: EXCLUDE_TASKS,
includeTasks: INCLUDE_TASKS,
additionalConfig: ADDITIONAL_CONFIG,
dataStorageSize: DATA_NODE_STORAGE,
mlStorageSize: ML_NODE_STORAGE,
Expand Down Expand Up @@ -360,6 +378,9 @@ pipeline {
dataInstanceType: DATA_INSTANCE_TYPE,
userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false",
workloadParams: WORKLOAD_PARAMS,
testProcedure: TEST_PROCEDURE,
excludeTasks: EXCLUDE_TASKS,
includeTasks: INCLUDE_TASKS,
additionalConfig: ADDITIONAL_CONFIG,
dataStorageSize: DATA_NODE_STORAGE,
mlStorageSize: ML_NODE_STORAGE,
Expand Down
7 changes: 5 additions & 2 deletions tests/jenkins/TestRunBenchmarkTestScriptDistributionUrl.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest {
void setUp() {
helper.registerSharedLibrary(
library().name('jenkins')
.defaultVersion('5.10.2')
.defaultVersion('5.11.0')
.allowOverride(true)
.implicit(true)
.targetPath('vars')
Expand Down Expand Up @@ -76,6 +76,9 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest {
binding.setVariable('ENABLE_REMOTE_STORE', 'false')
binding.setVariable('USER_TAGS', 'run-type:test')
binding.setVariable('WORKLOAD_PARAMS', '')
binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts')
binding.setVariable('EXCLUDE_TASKS', '')
binding.setVariable('INCLUDE_TASKS', '')
binding.setVariable('ADDITIONAL_CONFIG', '')
binding.setVariable('CLIENT_NODE_COUNT', '')
binding.setVariable('INGEST_NODE_COUNT', '')
Expand Down Expand Up @@ -128,7 +131,7 @@ class TestRunBenchmarkTestScriptDistributionUrl extends BuildPipelineTest {

assertThat(testScriptCommands.size(), equalTo(1))
assertThat(testScriptCommands, hasItems(
"./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --data-node-storage 100 ".toString()
"./test.sh benchmark-test --distribution-url https://artifacts.com/artifact.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --data-node-storage 100 ".toString()
))
}

Expand Down
7 changes: 5 additions & 2 deletions tests/jenkins/TestRunNonSecBenchmarkTestScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{
void setUp() {
helper.registerSharedLibrary(
library().name('jenkins')
.defaultVersion('5.10.2')
.defaultVersion('5.11.0')
.allowOverride(true)
.implicit(true)
.targetPath('vars')
Expand Down Expand Up @@ -76,6 +76,9 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{
binding.setVariable('ENABLE_REMOTE_STORE', 'false')
binding.setVariable('USER_TAGS', 'run-type:test')
binding.setVariable('WORKLOAD_PARAMS', '')
binding.setVariable('TEST_PROCEDURE', 'append-no-conflicts')
binding.setVariable('EXCLUDE_TASKS', 'type:search,scroll')
binding.setVariable('INCLUDE_TASKS', 'type:search,scroll')
binding.setVariable('ADDITIONAL_CONFIG', '')
binding.setVariable('CLIENT_NODE_COUNT', '')
binding.setVariable('INGEST_NODE_COUNT', '')
Expand Down Expand Up @@ -137,7 +140,7 @@ class TestRunNonSecBenchmarkTestScript extends BuildPipelineTest{

assertThat(testScriptCommands.size(), equalTo(1))
assertThat(testScriptCommands, hasItem(
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --data-node-storage 100 ".toString()
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-non-security-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --suffix 307 --manager-node-count 3 --data-node-count 3 --data-instance-type r5-4xlarge --test-procedure append-no-conflicts --exclude-tasks type:search,scroll --include-tasks type:search,scroll --data-node-storage 100 ".toString()
))
}

Expand Down
9 changes: 6 additions & 3 deletions tests/jenkins/TestRunSecureBenchmarkTestScript.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{
void setUp() {
helper.registerSharedLibrary(
library().name('jenkins')
.defaultVersion('5.10.2')
.defaultVersion('5.11.0')
.allowOverride(true)
.implicit(true)
.targetPath('vars')
Expand Down Expand Up @@ -76,6 +76,9 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{
binding.setVariable('DATA_NODE_COUNT', '3')
binding.setVariable('USER_TAGS', 'run-type:test')
binding.setVariable('WORKLOAD_PARAMS', '')
binding.setVariable('TEST_PROCEDURE', '')
binding.setVariable('EXCLUDE_TASKS', '')
binding.setVariable('INCLUDE_TASKS', '')
binding.setVariable('ADDITIONAL_CONFIG', '')
binding.setVariable('CLIENT_NODE_COUNT', '')
binding.setVariable('INGEST_NODE_COUNT', '')
Expand Down Expand Up @@ -138,8 +141,8 @@ class TestRunSecureBenchmarkTestScript extends BuildPipelineTest{

assertThat(testScriptCommands.size(), equalTo(2))
assertThat(testScriptCommands, hasItems(
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(),
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString(),
"./test.sh benchmark-test --bundle-manifest tests/jenkins/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc-taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,run-type:test,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-segment-replication-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --data-node-storage 100 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
))
}

Expand Down
Loading

0 comments on commit 44a68e4

Please sign in to comment.