From 747aa97592a1b0350471871bda81c454b6da3bba Mon Sep 17 00:00:00 2001
From: Chris Llanwarne <cjllanwarne@users.noreply.github.com>
Date: Fri, 10 May 2019 14:17:30 -0400
Subject: [PATCH] Call cache no copy fixups and centaur error improvements
 (#4951)

---
 .../centaur/reporting/Slf4jReporter.scala     |  5 +++--
 .../invalidate_bad_caches_jes_no_copy.test    |  3 +++
 .../src/main/scala/centaur/test/Test.scala    | 20 ++++++++++++++-----
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala
index c0be908f26f..a5da61eacbc 100644
--- a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala
+++ b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala
@@ -24,9 +24,10 @@ class Slf4jReporter(override val params: ErrorReporterParams)
     IO {
       val message =
         s"Test '${testEnvironment.name}' " +
-          centaurTestException.workflowIdOption.map("with workflow id '" + _ + "' ").getOrElse("") +
           s"failed on attempt ${testEnvironment.attempt + 1} " +
-          s"of ${testEnvironment.retries + 1}"
+          s"of ${testEnvironment.retries + 1} " +
+          centaurTestException.workflowIdOption.map("with workflow id '" + _ + "' ").getOrElse("")
+
       logger.error(message, centaurTestException)
     }
   }
diff --git a/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test b/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test
index 2b22b9096af..b2986be4b12 100644
--- a/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test
+++ b/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_jes_no_copy.test
@@ -2,6 +2,9 @@ name: invalidate_bad_caches_jes_no_copy
 testFormat: workflowsuccess
 backends: [Papi-Caching-No-Copy]
 
+# No point retrying failures since they'll just end up colliding with previous results:
+retryTestFailures: false
+
 files {
   workflow: invalidate_bad_caches/invalidate_bad_caches_no_copy.wdl
 }
diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala
index 77400c7a668..89a528c1e97 100644
--- a/centaur/src/main/scala/centaur/test/Test.scala
+++ b/centaur/src/main/scala/centaur/test/Test.scala
@@ -6,11 +6,10 @@ import cats.Monad
 import cats.effect.IO
 import cats.instances.list._
 import cats.syntax.traverse._
-import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur._
 import centaur.api.CentaurCromwellClient
-import centaur.api.CentaurCromwellClient.LogFailures
 import centaur.test.metadata.WorkflowFlatMetadata
+import centaur.test.metadata.WorkflowFlatMetadata._
 import centaur.test.submit.SubmitHttpResponse
 import centaur.test.workflow.Workflow
 import com.google.api.services.genomics.Genomics
@@ -22,9 +21,10 @@ import com.typesafe.config.Config
 import common.validation.Validation._
 import configs.syntax._
 import cromwell.api.CromwellClient.UnsuccessfulRequestException
-import cromwell.api.model.{CallCacheDiff, Failed, SubmittedWorkflow, TerminalStatus, WorkflowId, WorkflowMetadata, WorkflowStatus}
+import cromwell.api.model.{CallCacheDiff, Failed, SubmittedWorkflow, Succeeded, TerminalStatus, WorkflowId, WorkflowMetadata, WorkflowStatus}
 import cromwell.cloudsupport.gcp.GoogleConfiguration
 import cromwell.cloudsupport.gcp.auth.GoogleAuthMode
+import io.circe.parser._
 import spray.json.JsString
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -204,6 +204,7 @@ object Operations {
 
 
       override def run: IO[SubmittedWorkflow] = status(timeout).timeout(CentaurConfig.maxWorkflowLength)
+
     }
   }
 
@@ -223,7 +224,17 @@ object Operations {
             case s if s == expectedStatus => IO.pure(workflow)
             case s: TerminalStatus =>
               CentaurCromwellClient.metadata(workflow) flatMap { metadata =>
-                val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id})"
+                val failuresString = if (expectedStatus == Succeeded) {
+                  (for {
+                    metadataJson <- parse(metadata.value).toOption
+                    asObject <- metadataJson.asObject
+                    failures <- asObject.toMap.get("failures")
+                  } yield s" Metadata 'failures' content: ${failures.spaces2}").getOrElse("No additional failure information found in metadata.")
+                } else {
+                  ""
+                }
+
+                val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString"
                 IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata))
               }
             case _ => for {
@@ -386,7 +397,6 @@ object Operations {
         validateMetadata(workflow, expectedMetadata).handleErrorWith({ _ =>
           for {
             _ <- IO.sleep(2.seconds)
-            _ = if (LogFailures) Console.err.println(s"Metadata mismatch for ${submittedWorkflow.id} - retrying")
             recurse <- eventuallyMetadata(workflow, expectedMetadata)
           } yield recurse
         })