Skip to content

Commit 6738975

Browse files
AlexITCaednichols
AlexITC
and
aednichols
authored
WX-1557 Add more tests to the GCP Batch backend (#7394)
Co-authored-by: Adam Nichols <anichols@broadinstitute.org>
1 parent 69d8254 commit 6738975

File tree

3 files changed

+90
-24
lines changed

3 files changed

+90
-24
lines changed

‎supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,11 @@ import _root_.wdl.draft2.model._
66
import akka.actor.{ActorRef, Props}
77
import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe}
88
import cats.data.NonEmptyList
9+
import cloud.nio.impl.drs.DrsCloudNioFileProvider.DrsReadInterpreter
10+
import cloud.nio.impl.drs.{DrsCloudNioFileSystemProvider, GoogleOauthDrsCredentials}
911
import com.google.cloud.NoCredentials
1012
import com.google.cloud.batch.v1.{Job, JobName}
13+
import com.typesafe.config.{Config, ConfigFactory}
1114
import common.collections.EnhancedCollections._
1215
import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse
1316
import cromwell.backend._
@@ -56,6 +59,7 @@ import scala.concurrent.duration._
5659
import scala.concurrent.{Await, ExecutionContext, Future, Promise}
5760
import scala.language.postfixOps
5861
import common.mock.MockSugar
62+
import cromwell.filesystems.drs.DrsPathBuilder
5963
import org.mockito.Mockito._
6064

6165
class GcpBatchAsyncBackendJobExecutionActorSpec
@@ -396,6 +400,55 @@ class GcpBatchAsyncBackendJobExecutionActorSpec
396400
GcpBatchAsyncBackendJobExecutionActor.groupParametersByGcsBucket(inputs) shouldEqual expected
397401
}
398402

403+
it should "generate a CSV manifest for DRS inputs, ignoring non-DRS inputs" in {
404+
def makeDrsPathBuilder: DrsPathBuilder = {
405+
val drsResolverConfig: Config = ConfigFactory.parseString(
406+
"""resolver {
407+
| url = "http://drshub-url"
408+
|}
409+
|""".stripMargin
410+
)
411+
412+
val fakeCredentials = NoCredentials.getInstance
413+
414+
val drsReadInterpreter: DrsReadInterpreter = (_, _) =>
415+
throw new UnsupportedOperationException(
416+
"PipelinesApiAsyncBackendJobExecutionActorSpec doesn't need to use drs read interpreter."
417+
)
418+
419+
DrsPathBuilder(
420+
new DrsCloudNioFileSystemProvider(drsResolverConfig,
421+
GoogleOauthDrsCredentials(fakeCredentials, 1.minutes),
422+
drsReadInterpreter
423+
),
424+
None
425+
)
426+
}
427+
428+
val mount = GcpBatchWorkingDisk(DiskType.LOCAL, 1)
429+
430+
def makeDrsInput(name: String, drsUri: String, containerPath: String): GcpBatchFileInput = {
431+
val drsPath = makeDrsPathBuilder.build(drsUri).get
432+
val containerRelativePath = DefaultPathBuilder.get(containerPath)
433+
GcpBatchFileInput(name, drsPath, containerRelativePath, mount)
434+
}
435+
436+
val nonDrsInput: GcpBatchFileInput = GcpBatchFileInput("nnn",
437+
DefaultPathBuilder.get("/local/nnn.bai"),
438+
DefaultPathBuilder.get("/path/to/nnn.bai"),
439+
mount
440+
)
441+
442+
val inputs = List(
443+
makeDrsInput("aaa", "drs://drs.example.org/aaa", "path/to/aaa.bai"),
444+
nonDrsInput,
445+
makeDrsInput("bbb", "drs://drs.example.org/bbb", "path/to/bbb.bai")
446+
)
447+
448+
GcpBatchAsyncBackendJobExecutionActor.generateDrsLocalizerManifest(inputs) shouldEqual
449+
"drs://drs.example.org/aaa,/mnt/disks/cromwell_root/path/to/aaa.bai\r\ndrs://drs.example.org/bbb,/mnt/disks/cromwell_root/path/to/bbb.bai\r\n"
450+
}
451+
399452
it should "send proper value for \"number of reference files used gauge\" metric, or don't send anything if reference disks feature is disabled" in {
400453

401454
val expectedInput1 = GcpBatchFileInput(name = "testfile1",

‎supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,9 +124,14 @@ class GcpBatchConfigurationSpec
124124
).root shouldBe "gs://my-cromwell-workflows-bucket"
125125
}
126126

127-
// it should "have correct docker" in {
128-
// val dockerConf = new GcpBatchConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig), googleConfiguration, batchAttributes).dockerCredentials
129-
// dockerConf shouldBe defined
130-
// dockerConf.get.token shouldBe "dockerToken"
131-
// }
127+
it should "have correct docker" in {
128+
val dockerConf = new GcpBatchConfiguration(
129+
BackendConfigurationDescriptor(backendConfig, globalConfig),
130+
googleConfiguration,
131+
batchAttributes
132+
).dockerCredentials
133+
134+
dockerConf shouldBe defined
135+
dockerConf.get.token shouldBe "dockerToken"
136+
}
132137
}

‎supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,9 @@ package cromwell.backend.google.batch.models
22

33
import cats.data.NonEmptyList
44
import cromwell.backend.RuntimeAttributeDefinition
5+
import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk, GcpBatchWorkingDisk}
56
import cromwell.backend.google.batch.models.GcpBatchTestConfig._
67
import cromwell.backend.validation.ContinueOnReturnCodeSet
7-
//import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk}
8-
import cromwell.backend.google.batch.io.{DiskType, GcpBatchWorkingDisk}
98
import cromwell.core.WorkflowOptions
109
import eu.timepit.refined.refineMV
1110
import org.scalatest.TestSuite
@@ -48,9 +47,8 @@ final class GcpBatchRuntimeAttributesSpec
4847
}
4948

5049
"fail to validate an invalid Docker entry" in {
51-
pending
5250
val runtimeAttributes = Map("docker" -> WomInteger(1))
53-
assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be String")
51+
assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be a String")
5452
}
5553

5654
"validate a valid failOnStderr entry" in {
@@ -152,21 +150,31 @@ final class GcpBatchRuntimeAttributesSpec
152150
)
153151
}
154152

155-
// "validate a valid disks entry" in {
156-
// val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomString("local-disk 20 SSD"))
157-
// val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(GcpBatchAttachedDisk.parse("local-disk 20 SSD").get))
158-
// assertBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
159-
// }
160-
161-
// "fail to validate an invalid disks entry" in {
162-
// val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomInteger(10))
163-
// assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disks runtime attribute to be a comma separated String or Array[String]")
164-
// }
165-
166-
// "fail to validate a valid disks array entry" in {
167-
// val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomArray(WomArrayType(WomStringType), List(WomString("blah"), WomString("blah blah"))))
168-
// assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'")
169-
// }
153+
"validate a valid disks entry" in {
154+
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomString("local-disk 20 SSD"))
155+
val expectedRuntimeAttributes =
156+
expectedDefaults.copy(disks = Seq(GcpBatchAttachedDisk.parse("local-disk 20 SSD").get))
157+
assertBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes)
158+
}
159+
160+
"fail to validate an invalid disks entry" in {
161+
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomInteger(10))
162+
assertBatchRuntimeAttributesFailedCreation(
163+
runtimeAttributes,
164+
"Expecting disks runtime attribute to be a comma separated String or Array[String]"
165+
)
166+
}
167+
168+
"fail to validate a valid disks array entry" in {
169+
val runtimeAttributes =
170+
Map("docker" -> WomString("ubuntu:latest"),
171+
"disks" -> WomArray(WomArrayType(WomStringType), List(WomString("blah"), WomString("blah blah")))
172+
)
173+
assertBatchRuntimeAttributesFailedCreation(
174+
runtimeAttributes,
175+
"Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'"
176+
)
177+
}
170178

171179
"validate a valid memory entry" in {
172180
val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "memory" -> WomString("1 GB"))

0 commit comments

Comments
 (0)