BREAKING CHANGE: group tests now accepts multiple tests and multiple …

…remotes

justfile

@@ -65,20 +65,23 @@ clean-buckets profile: _setup
 test remote test_name folder=date: _setup
   @just _test-k6 {{remote}} {{test_name}} {{folder}}
 
-# Run the same test on multiple remotes
-group-test test_name group_name +remotes: _setup
+# Run on multiple(comma separated) remotes, multiple tests
+group-test group_name remote_names test_names: _setup
   #!/usr/bin/env sh
   results_subdir={{group_name}}/{{date}}
   results_dir={{results_prefix}}/$results_subdir
   echo $results_dir
   mkdir -p $results_dir
-  # TODO: do we want to parallelize?
-  # parallel_processes=2
-  # printf "%s\0" {{remotes}} | xargs -0 -I @ -P $parallel_processes just test @ {{test_name}}
-  for remote in {{remotes}}; do
-    just test $remote {{test_name}} {{date}} || true
+  remotes=$(echo {{remote_names}} | tr ',' ' ')
+  tests=$(echo {{test_names}} | tr ',' ' ')
+  for remote in $remotes; do
+    for test_name in $tests; do
+      echo "STARTING $test_name on $remote"
+      just test $remote $test_name {{date}} || true
+    done
   done
-  echo {{remotes}} > $results_dir/remotes.txt
+  echo $remotes > $results_dir/remotes.txt
+  echo $tests > $results_dir/tests.txt
   just write-reports {{group_name}} {{date}}
 
 # Create html reports based on results from the past minutes

run_tests.sh

@@ -7,16 +7,21 @@ cd /app
 touch results/debug_last_test_start
 
 # space separated list of remote names
-remotes=$(dasel -f ./config.yaml -r yaml -s  '.remotes.all().key()' | grep -v '\-second$' | tr '\n' ' ')
+remotes=$(dasel -f ./config.yaml -r yaml -s  '.remotes.all().key()' | grep -v '\-second$' | tr '\n' ',')
 small_set_remotes=$(dasel -f ./config.yaml -r yaml -s  '.small_set_remotes')
 
 # Check if TEST_SUITE is set and execute commands accordingly
 if [ "$TEST_SUITE" = "big" ]; then
     # Run tests for big suite
-    just group-test index-s3 _big "$remotes" > results/debug_last_test.txt 2>&1
+    #tests="index-s3"
+    tests="boto3-presigned,aws-cli-presigned,aws-cli-create-bucket,rclone-s3,aws-cli-s3-acl,k6-jslib-objects,k6-jslib-buckets,delete-objects,aws-cli-multipart"
+    # TODO: return the mgc-s3, test
+    just group-test _big "$remotes" "$tests" > results/debug_last_big_test.txt 2>&1
 elif [ "$TEST_SUITE" = "small" ]; then
     # Run tests for small suite
+    tests="delete-objects,k6-jslib-buckets"
     just group-test delete-objects _small "$small_set_remotes" > results/debug_last_test.txt 2>&1
+    just group-test _big "$remotes" "$tests" > results/debug_last_big_test.txt 2>&1
 else
     echo "Unknown TEST_SUITE value: $TEST_SUITE" > results/debug_last_test.txt 2>&1
     exit 1

src/k6/utils/clis.js

@@ -5,7 +5,7 @@ const profileName2 = `${__ENV.AWS_CLI_PROFILE}-second`
 
 export function aws({endpoint, region}, subCommand, args=[]){
   console.log(`execute aws`)
-  console.log(`aws ${subCommand} --profile ${profileName} --endpoint ${endpoint} --region ${region} ${args.join(' ')}`)
+  console.log(`aws ${subCommand} --profile ${profileName} --region ${region} ${args.join(' ')}`)
   return exec.command("aws", [
     subCommand,
     "--profile", profileName,