WX-1351 Speed up Centaur Horicromtal PapiV2 Beta (#7329)

Author: aednichols

core/src/main/resources/reference.conf

@@ -118,7 +118,8 @@ system {
   max-concurrent-workflows = 5000
 
   # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist
-  max-workflow-launch-count = 50
+  # Deviating from 1 is not recommended for multi-runner setups due to possible deadlocks. [BW-962]
+  max-workflow-launch-count = 1
 
   # Workflows will be grouped by the value of the specified field in their workflow options.
   #

src/ci/bin/test.inc.sh

@@ -1607,6 +1607,45 @@ cromwell::build::generate_code_coverage() {
     fi
 }
 
+cromwell::build::print_workflow_statistics() {
+    echo "Total workflows"
+    mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \
+        "SELECT COUNT(*) as total_workflows_run FROM WORKFLOW_METADATA_SUMMARY_ENTRY;"
+
+    echo "Late starters"
+    mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \
+        "SELECT WORKFlOW_NAME as name,
+            TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes,
+            START_TIMESTAMP as START,
+            END_TIMESTAMP as end
+        FROM WORKFLOW_METADATA_SUMMARY_ENTRY
+            WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL # exclude subworkflows
+            ORDER BY START_TIMESTAMP DESC
+            LIMIT 20;"
+
+    echo "Late finishers"
+    mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \
+        "SELECT WORKFlOW_NAME as name,
+            TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as runtime_minutes,
+            START_TIMESTAMP as start,
+            END_TIMESTAMP as END
+        FROM WORKFLOW_METADATA_SUMMARY_ENTRY
+            WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL
+            ORDER BY END_TIMESTAMP DESC
+            LIMIT 20;"
+
+    echo "Long duration"
+    mysql --host=127.0.0.1 --user=cromwell --password=test cromwell_test -e \
+        "SELECT WORKFlOW_NAME as name,
+            TIMESTAMPDIFF(MINUTE, START_TIMESTAMP, END_TIMESTAMP) as RUNTIME_MINUTES,
+            START_TIMESTAMP as start,
+            END_TIMESTAMP as end
+        FROM WORKFLOW_METADATA_SUMMARY_ENTRY
+            WHERE PARENT_WORKFLOW_EXECUTION_UUID IS NULL
+            ORDER BY RUNTIME_MINUTES DESC
+            LIMIT 20;"
+}
+
 cromwell::build::exec_retry_function() {
     local retried_function
     local retry_count

src/ci/bin/testCentaurHoricromtalPapiV2beta.sh

@@ -19,11 +19,13 @@ cromwell::build::assemble_jars
 cromwell::build::build_cromwell_docker
 
 cromwell::build::run_centaur \
-    -p 100 \
+    -p 500 \
     -e localdockertest \
     -e relative_output_paths \
     -e relative_output_paths_colliding \
     -e standard_output_paths_colliding_prevented \
     -e papi_v2alpha1_gcsa \
 
 cromwell::build::generate_code_coverage
+
+cromwell::build::print_workflow_statistics

src/ci/bin/testCentaurPapiV2beta.sh

@@ -25,3 +25,5 @@ cromwell::build::run_centaur \
     -e papi_v2alpha1_gcsa \
 
 cromwell::build::generate_code_coverage
+
+cromwell::build::print_workflow_statistics

src/ci/resources/papi_v2beta_horicromtal_application.conf

@@ -1,8 +1,8 @@
 include "papi_v2beta_application.conf"
 
-system.max-workflow-launch-count=1
-system.new-workflow-poll-rate=10
-system.max-concurrent-workflows=30
+system.max-workflow-launch-count=10
+system.new-workflow-poll-rate=1
+system.max-concurrent-workflows=500
 system.cromwell_id_random_suffix=false
 
 # Turn off token logging to reduce log volume