Skip to content

Add docker-composed based test fixture for GCS #48762

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 7, 2019
Merged

Add docker-composed based test fixture for GCS #48762

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8a82714
Add docker-composed based test fixture for GCS
tlrx Oct 31, 2019
65bc9e5
Merge branch 'master' into docker-compose-fixture-for-gcs
tlrx Nov 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions plugins/repository-gcs/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ dependencies {
compile 'io.opencensus:opencensus-api:0.18.0'
compile 'io.opencensus:opencensus-contrib-http-util:0.18.0'
compile 'com.google.apis:google-api-services-storage:v1-rev20190426-1.28.0'

testCompile project(':test:fixtures:gcs-fixture')
}

dependencyLicenses {
Expand Down
61 changes: 24 additions & 37 deletions plugins/repository-gcs/qa/google-cloud-storage/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.info.BuildParams
import org.elasticsearch.gradle.test.AntFixture

import java.nio.file.Files
import java.security.KeyPair
Expand All @@ -30,12 +29,14 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE

apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
apply plugin: 'elasticsearch.test.fixtures'

// TODO think about flattening qa:google-cloud-storage project into parent
dependencies {
testCompile project(path: ':plugins:repository-gcs')
}

testFixtures.useFixture(':test:fixtures:gcs-fixture')
boolean useFixture = false

String gcsServiceAccount = System.getenv("google_storage_service_account")
Expand All @@ -45,7 +46,7 @@ String gcsBasePath = System.getenv("google_storage_base_path")
File serviceAccountFile = null
if (!gcsServiceAccount && !gcsBucket && !gcsBasePath) {
serviceAccountFile = new File(project.buildDir, 'generated-resources/service_account_test.json')
gcsBucket = 'bucket_test'
gcsBucket = 'bucket'
gcsBasePath = 'integration_test'
useFixture = true
} else if (!gcsServiceAccount || !gcsBucket || !gcsBasePath) {
Expand All @@ -58,12 +59,11 @@ def encodedCredentials = {
Base64.encoder.encodeToString(Files.readAllBytes(serviceAccountFile.toPath()))
}

/** A task to start the GoogleCloudStorageFixture which emulates a Google Cloud Storage service **/
task googleCloudStorageFixture(type: AntFixture) {
dependsOn testClasses
env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = "${BuildParams.runtimeJavaHome}/bin/java"
args 'org.elasticsearch.repositories.gcs.GoogleCloudStorageFixture', baseDir, 'bucket_test'
def fixtureAddress = { fixture ->
assert useFixture : 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:gcs-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}

/** A service account file that points to the Google Cloud Storage service emulated by the fixture **/
Expand All @@ -87,6 +87,19 @@ task createServiceAccountFile() {
}

task thirdPartyTest (type: Test) {
if (useFixture) {
thirdPartyTest.dependsOn createServiceAccountFile
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if dependsOn tasks.postProcessFixture is also needed?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope. The test fixtures plugin automatically makes all Test tasks depend on the fixture tasks. Essentially, when a project "uses a fixture" it's assumed that all tests require that fixture.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, thanks @mark-vieira !

nonInputProperties.systemProperty 'test.google.endpoint', "${ -> fixtureAddress('gcs-fixture-third-party') }"
nonInputProperties.systemProperty 'test.google.tokenURI', "${ -> fixtureAddress('gcs-fixture-third-party') }/o/oauth2/token"

gradle.taskGraph.whenReady {
if (it.hasTask(gcsThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables " +
"'google_storage_service_account', 'google_storage_bucket', 'google_storage_base_path' are set.")
}
}
}

include '**/GoogleCloudStorageThirdPartyTests.class'
systemProperty 'tests.security.manager', false
systemProperty 'test.google.bucket', gcsBucket
Expand All @@ -98,32 +111,6 @@ task gcsThirdPartyTests {
dependsOn check
}

if (useFixture) {
// TODO think about running the fixture in the same JVM as tests
thirdPartyTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
thirdPartyTest.finalizedBy googleCloudStorageFixture.getStopTask()

def fixtureEndpoint = {
"http://${googleCloudStorageFixture.addressAndPort}"
}

def tokenURI = {
"http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token"
}

thirdPartyTest {
nonInputProperties.systemProperty 'test.google.endpoint', "${ -> fixtureEndpoint.call() }"
nonInputProperties.systemProperty 'test.google.tokenURI', "${ -> tokenURI.call() }"
}

gradle.taskGraph.whenReady {
if (it.hasTask(gcsThirdPartyTests)) {
throw new IllegalStateException("Tried to run third party tests but not all of the necessary environment variables 'google_storage_service_account', " +
"'google_storage_bucket', 'google_storage_base_path' are set.")
}
}
}

integTest.mustRunAfter(thirdPartyTest)
check.dependsOn thirdPartyTest

Expand All @@ -147,10 +134,10 @@ testClusters.integTest {
keystore 'gcs.client.integration_test.credentials_file', serviceAccountFile, IGNORE_VALUE

if (useFixture) {
tasks.integTest.dependsOn createServiceAccountFile, googleCloudStorageFixture
tasks.integTest.dependsOn createServiceAccountFile
/* Use a closure on the string to delay evaluation until tests are executed */
setting 'gcs.client.integration_test.endpoint', { "http://${googleCloudStorageFixture.addressAndPort}" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "http://${googleCloudStorageFixture.addressAndPort}/o/oauth2/token" }, IGNORE_VALUE
setting 'gcs.client.integration_test.endpoint', { "${ -> fixtureAddress('gcs-fixture') }" }, IGNORE_VALUE
setting 'gcs.client.integration_test.token_uri', { "${ -> fixtureAddress('gcs-fixture') }/o/oauth2/token" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-gcs plugin"
}
Expand Down
Loading