Skip to content

Commit 610995b

Browse files
THWisemanAlexITC
THWiseman
authored and
AlexITC
committed
[WX-1345] Automatic Token Acquisition for TES Config (#7256)
1 parent 52c0b79 commit 610995b

File tree

7 files changed

+197
-30
lines changed

7 files changed

+197
-30
lines changed

‎.gitignore‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,10 @@ tesk_application.conf
5858
**/venv/
5959
exome_germline_single_sample_v1.3/
6060
**/*.pyc
61+
src/ci/resources/*.temp
6162

6263
# GHA credentials
6364
gha-creds-*.json
65+
66+
# jenv
67+
.java-version

‎project/ContinuousIntegration.scala‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ object ContinuousIntegration {
99
lazy val ciSettings: Seq[Setting[_]] = List(
1010
srcCiResources := sourceDirectory.value / "ci" / "resources",
1111
targetCiResources := target.value / "ci" / "resources",
12+
envFile := srcCiResources.value / "env.temp", //generated by resources/acquire_b2c_token.sh
13+
1214
vaultToken := userHome / ".vault-token",
1315
copyCiResources := {
1416
IO.copyDirectory(srcCiResources.value, targetCiResources.value)
@@ -26,7 +28,12 @@ object ContinuousIntegration {
2628
if (vaultToken.value.isDirectory) {
2729
sys.error(s"""The vault token file "${vaultToken.value}" should not be a directory.""")
2830
}
29-
val cmd = List(
31+
32+
// Only include the local file argument if the file exists (local development w/ acquire_b2c_token.sh)
33+
// Don't include it otherwise (CI/CD and other development)
34+
val localEnvFileArgs = if(envFile.value.exists()) List("-e", s"ENV_FILE=${envFile.value}") else List()
35+
36+
val cmd: List[String] = List.concat(List(
3037
"docker",
3138
"run",
3239
"--rm",
@@ -35,7 +42,9 @@ object ContinuousIntegration {
3542
"-v",
3643
s"${srcCiResources.value}:${srcCiResources.value}",
3744
"-v",
38-
s"${targetCiResources.value}:${targetCiResources.value}",
45+
s"${targetCiResources.value}:${targetCiResources.value}"),
46+
localEnvFileArgs,
47+
List(
3948
"-e",
4049
"ENVIRONMENT=not_used",
4150
"-e",
@@ -44,7 +53,7 @@ object ContinuousIntegration {
4453
s"OUT_PATH=${targetCiResources.value}",
4554
"broadinstitute/dsde-toolbox:dev",
4655
"render-templates.sh"
47-
)
56+
))
4857
val result = cmd ! log
4958
if (result != 0) {
5059
sys.error(
@@ -70,6 +79,7 @@ object ContinuousIntegration {
7079
private val srcCiResources: SettingKey[File] = settingKey[File]("Source directory for CI resources")
7180
private val targetCiResources: SettingKey[File] = settingKey[File]("Target directory for CI resources")
7281
private val vaultToken: SettingKey[File] = settingKey[File]("File with the vault token")
82+
private val envFile: SettingKey[File] = settingKey[File]("File with the environment variables needed to render CI resources.")
7383

7484
/**
7585
* For "reasons" these projects are excluded from the root aggregation in build.sbt.

‎runConfigurations/Repo template_ Cromwell server TES.run.xml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,24 @@
1919
<option name="RunConfigurationTask" enabled="true" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
2020
</method>
2121
</configuration>
22+
<configuration default="false" name="Repo template: Cromwell server TES" type="Application" factoryName="Application">
23+
<option name="ALTERNATIVE_JRE_PATH" value="$PROJECT_DIR$/../.sdkman/candidates/java/current" />
24+
<envs>
25+
<env name="CROMWELL_BUILD_CENTAUR_SLICK_PROFILE" value="slick.jdbc.MySQLProfile$" />
26+
<env name="CROMWELL_BUILD_CENTAUR_JDBC_DRIVER" value="com.mysql.cj.jdbc.Driver" />
27+
<env name="CROMWELL_BUILD_CENTAUR_JDBC_URL" value="jdbc:mysql://localhost:3306/cromwell_test?allowPublicKeyRetrieval=true&amp;useSSL=false&amp;rewriteBatchedStatements=true&amp;serverTimezone=UTC&amp;useInformationSchema=true" />
28+
<env name="CROMWELL_BUILD_RESOURCES_DIRECTORY" value="target/ci/resources" />
29+
<env name="CROMWELL_BUILD_PAPI_JSON_FILE" value="target/ci/resources/cromwell-centaur-service-account.json" />
30+
<env name="CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT" value="128000" />
31+
<env name="CROMWELL_BUILD_CENTAUR_256_BITS_KEY" value="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" />
32+
</envs>
33+
<option name="MAIN_CLASS_NAME" value="cromwell.CromwellApp" />
34+
<module name="cromwell" />
35+
<option name="PROGRAM_PARAMETERS" value="server" />
36+
<option name="VM_PARAMETERS" value="-Dconfig.file=target/ci/resources/tes_application.conf" />
37+
<method v="2">
38+
<option name="Make" enabled="true" />
39+
<option name="RunConfigurationTask" enabled="false" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
40+
</method>
41+
</configuration>
2242
</component>

‎runConfigurations/Repo template_ Cromwell server Terra TES.run.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<component name="ProjectRunConfigurationManager">
2+
<configuration default="false" name="Repo template: Cromwell server Terra TES" type="Application" factoryName="Application">
3+
<option name="ALTERNATIVE_JRE_PATH" value="$PROJECT_DIR$/../.sdkman/candidates/java/current" />
4+
<envs>
5+
<env name="CROMWELL_BUILD_CENTAUR_SLICK_PROFILE" value="slick.jdbc.MySQLProfile$" />
6+
<env name="CROMWELL_BUILD_CENTAUR_JDBC_DRIVER" value="com.mysql.cj.jdbc.Driver" />
7+
<env name="CROMWELL_BUILD_CENTAUR_JDBC_URL" value="jdbc:mysql://localhost:3306/cromwell_test?allowPublicKeyRetrieval=true&amp;useSSL=false&amp;rewriteBatchedStatements=true&amp;serverTimezone=UTC&amp;useInformationSchema=true" />
8+
<env name="CROMWELL_BUILD_RESOURCES_DIRECTORY" value="target/ci/resources" />
9+
<env name="CROMWELL_BUILD_PAPI_JSON_FILE" value="target/ci/resources/cromwell-centaur-service-account.json" />
10+
<env name="CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT" value="128000" />
11+
<env name="CROMWELL_BUILD_CENTAUR_256_BITS_KEY" value="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" />
12+
</envs>
13+
<option name="MAIN_CLASS_NAME" value="cromwell.CromwellApp" />
14+
<module name="cromwell" />
15+
<option name="PROGRAM_PARAMETERS" value="server" />
16+
<option name="VM_PARAMETERS" value="-Dconfig.file=target/ci/resources/terra_tes_application.conf" />
17+
<method v="2">
18+
<option name="Make" enabled="true" />
19+
<option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_Acquire Google B2C Token" />
20+
<option name="RunConfigurationTask" enabled="true" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
21+
</method>
22+
</configuration>
23+
</component>

‎src/ci/resources/acquire_b2c_token.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
3+
# This script acquires a b2c token using gcloud auth and writes it to a file.
4+
# This file is used by renderCiResources to populate the CROMWELL_B2C_TOKEN in tes_application.conf.
5+
# It should be run as a "Before Launch" task in IntelliJ so the token is refreshed regularly.
6+
7+
### Setting up this script for the first time in IntelliJ?
8+
### 1. Open the "Run/Debug Configurations" dialog for the TES repo template.
9+
### 2. Add a new "Before Launch" task, selecting "Run External Tool".
10+
### This task should run BEFORE the renderCiResources task.
11+
### 3. Configure the new task:
12+
### - Name: "Acquire b2c token"
13+
### - Program: /full/path/to/src/ci/resources/acquire_b2c_token.sh
14+
### - Arguments: ./env.temp
15+
### - Working directory: /full/path/to/src/ci/resources
16+
### - Advanced Options > Uncheck "Synchronize files after execution"
17+
### 4. Click "OK" to save the new task. Apply and close the "Run/Debug Configurations" dialog.
18+
19+
# User must provide an output file path as an argument.
20+
# This can be provided by IntelliJ as a program argument in the run configuration.
21+
if [ -z "$1" ]; then
22+
echo "Error: Must specify an output file path for the environment file."
23+
exit 1
24+
fi
25+
26+
# Acquire a b2c token using gcloud auth.
27+
# Send stderr to /dev/null so that any gcloud auth errors/update prompts don't pollute the token output.
28+
echo "Using local gcloud auth to acquire a b2c token..."
29+
B2C_TOKEN=$(gcloud auth print-access-token 2>/dev/null)
30+
if [ $? -eq 0 ]; then
31+
echo "Acquired b2c token: ${B2C_TOKEN:0:4}****"
32+
else
33+
echo "Failed to acquire b2c token. Is your local shell logged into gcloud?"
34+
fi
35+
36+
# Create the output directory if it doesn't exist
37+
output_dir=$(dirname "$1")
38+
mkdir -p "$output_dir"
39+
40+
# Write the token to the output file. dsde-toolbox will call `source` on this file that we're writing,
41+
# so it's expected that this file contains valid bash syntax for setting an environment variable.
42+
echo "export CROMWELL_B2C_TOKEN=${B2C_TOKEN}" > "$1"

‎src/ci/resources/terra_tes_application.conf.ctmpl‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
include required(classpath("application.conf"))
2+
include "build_application.inc.conf"
3+
4+
filesystems {
5+
blob {
6+
class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
7+
global {
8+
class = "cromwell.filesystems.blob.BlobFileSystemManager"
9+
config {
10+
workspace-manager: {
11+
# static: Where is WSM Endpoint
12+
url: "https://workspace.dsde-dev.broadinstitute.org"
13+
# See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
14+
# This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
15+
b2cToken: "{{ env "CROMWELL_B2C_TOKEN" }}"
16+
}
17+
}
18+
}
19+
}
20+
}
21+
22+
engine {
23+
filesystems {
24+
local {
25+
enabled: true
26+
}
27+
http {
28+
enabled: true
29+
}
30+
blob {
31+
enabled: true
32+
}
33+
}
34+
}
35+
36+
37+
backend {
38+
default = "TES"
39+
providers {
40+
TES {
41+
actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
42+
config {
43+
# Root of a blob storage container to use
44+
# Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
45+
root = ""
46+
dockerRoot = "/cromwell-executions"
47+
# TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
48+
endpoint = ""
49+
concurrent-job-limit = 1000
50+
# Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
51+
workflow-execution-identity = ""
52+
# See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
53+
# This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
54+
bearer-token="Bearer {{ env "CROMWELL_B2C_TOKEN" }}"
55+
# Override default polling to make it faster for speedy tests
56+
poll-backoff {
57+
min: "10 seconds"
58+
max: "5 minutes"
59+
multiplier: 1.1
60+
randomization-factor: 0.5
61+
}
62+
63+
filesystems {
64+
blob {
65+
enabled: true
66+
}
67+
local {
68+
enabled: false
69+
}
70+
http {
71+
enabled: true
72+
}
73+
}
74+
}
75+
76+
# Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
77+
include "dockerhub_provider_config_v1.inc.conf"
78+
}
79+
}
80+
}

‎src/ci/resources/tes_application.conf‎

Lines changed: 15 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,26 +2,12 @@ include required(classpath("application.conf"))
22
include "build_application.inc.conf"
33

44
filesystems {
5-
blob {
6-
class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
7-
global {
8-
# One BFSM is shared across all BlobPathBuilders
9-
class = "cromwell.filesystems.blob.BlobFileSystemManager"
10-
config {
11-
container: "cromwell"
12-
endpoint: "https://<storage-account>.blob.core.windows.net"
13-
subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
14-
# WSM config is needed for accessing WSM-managed blob containers
15-
# created in Terra workspaces.
16-
workspace-manager {
17-
url: "https://workspace.dsde-dev.broadinstitute.org"
18-
workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
19-
container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
20-
b2cToken: "Zardoz"
5+
blob {
6+
class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
7+
global {
8+
class = "cromwell.filesystems.blob.BlobFileSystemManager"
219
}
22-
}
2310
}
24-
}
2511
}
2612

2713
engine {
@@ -33,7 +19,7 @@ engine {
3319
enabled: true
3420
}
3521
blob {
36-
enabled: false
22+
enabled: true
3723
}
3824
}
3925
}
@@ -45,27 +31,29 @@ backend {
4531
TES {
4632
actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
4733
config {
48-
# Use for running on blob storage
49-
#root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
50-
root = "cromwell-executions"
34+
# Root of a blob storage container to use
35+
# Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
36+
root = ""
5137
dockerRoot = "/cromwell-executions"
38+
# TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
5239
endpoint = "http://127.0.0.1:9000/v1/tasks"
5340
concurrent-job-limit = 1000
54-
41+
# Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
42+
workflow-execution-identity = ""
5543
# Override default polling to make it faster for speedy tests
5644
poll-backoff {
57-
min: "1 seconds"
45+
min: "10 seconds"
5846
max: "5 minutes"
5947
multiplier: 1.1
6048
randomization-factor: 0.5
6149
}
62-
50+
6351
filesystems {
6452
blob {
65-
enabled: false
53+
enabled: true
6654
}
6755
local {
68-
enabled: true
56+
enabled: false
6957
}
7058
http {
7159
enabled: true

0 commit comments

Comments
 (0)