Call cache no copy fixups and centaur error improvements (#4951)

broadinstitute · May 10, 2019 · 747aa97 · 747aa97
1 parent 668e521
commit 747aa97
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 7 deletions.
diff --git a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala
@@ -24,9 +24,10 @@ class Slf4jReporter(override val params: ErrorReporterParams)
     IO {
       val message =
         s"Test '${testEnvironment.name}' " +
-          centaurTestException.workflowIdOption.map("with workflow id '" + _ + "' ").getOrElse("") +
           s"failed on attempt ${testEnvironment.attempt + 1} " +
-          s"of ${testEnvironment.retries + 1}"
+          s"of ${testEnvironment.retries + 1} " +
+          centaurTestException.workflowIdOption.map("with workflow id '" + _ + "' ").getOrElse("")
+
       logger.error(message, centaurTestException)
     }
   }

diff --git a/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test b/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test
@@ -2,6 +2,9 @@ name: invalidate_bad_caches_jes_no_copy
 testFormat: workflowsuccess
 backends: [Papi-Caching-No-Copy]
 
+# No point retrying failures since they'll just end up colliding with previous results:
+retryTestFailures: false
+
 files {
   workflow: invalidate_bad_caches/invalidate_bad_caches_no_copy.wdl
 }

diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala
@@ -6,11 +6,10 @@ import cats.Monad
 import cats.effect.IO
 import cats.instances.list._
 import cats.syntax.traverse._
-import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur._
 import centaur.api.CentaurCromwellClient
-import centaur.api.CentaurCromwellClient.LogFailures
 import centaur.test.metadata.WorkflowFlatMetadata
+import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur.test.submit.SubmitHttpResponse
 import centaur.test.workflow.Workflow
 import com.google.api.services.genomics.Genomics
@@ -22,9 +21,10 @@ import com.typesafe.config.Config
 import common.validation.Validation._
 import configs.syntax._
 import cromwell.api.CromwellClient.UnsuccessfulRequestException
-import cromwell.api.model.{CallCacheDiff, Failed, SubmittedWorkflow, TerminalStatus, WorkflowId, WorkflowMetadata, WorkflowStatus}
+import cromwell.api.model.{CallCacheDiff, Failed, SubmittedWorkflow, Succeeded, TerminalStatus, WorkflowId, WorkflowMetadata, WorkflowStatus}
 import cromwell.cloudsupport.gcp.GoogleConfiguration
 import cromwell.cloudsupport.gcp.auth.GoogleAuthMode
+import io.circe.parser._
 import spray.json.JsString
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -204,6 +204,7 @@ object Operations {
 
 
       override def run: IO[SubmittedWorkflow] = status(timeout).timeout(CentaurConfig.maxWorkflowLength)
+
     }
   }
 
@@ -223,7 +224,17 @@ object Operations {
             case s if s == expectedStatus => IO.pure(workflow)
             case s: TerminalStatus =>
               CentaurCromwellClient.metadata(workflow) flatMap { metadata =>
-                val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id})"
+                val failuresString = if (expectedStatus == Succeeded) {
+                  (for {
+                    metadataJson <- parse(metadata.value).toOption
+                    asObject <- metadataJson.asObject
+                    failures <- asObject.toMap.get("failures")
+                  } yield s" Metadata 'failures' content: ${failures.spaces2}").getOrElse("No additional failure information found in metadata.")
+                } else {
+                  ""
+                }
+
+                val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString"
                 IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata))
               }
             case _ => for {
@@ -386,7 +397,6 @@ object Operations {
         validateMetadata(workflow, expectedMetadata).handleErrorWith({ _ =>
           for {
             _ <- IO.sleep(2.seconds)
-            _ = if (LogFailures) Console.err.println(s"Metadata mismatch for ${submittedWorkflow.id} - retrying")
             recurse <- eventuallyMetadata(workflow, expectedMetadata)
           } yield recurse
         })