Skip to content

Commit

Permalink
Add support for Secret Manager in Pub/Sub to Splunk Template.
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 433306801
  • Loading branch information
pranavbhandari24 authored and cloud-teleport committed Mar 8, 2022
1 parent ce9eb43 commit 48d521e
Show file tree
Hide file tree
Showing 7 changed files with 559 additions and 16 deletions.
17 changes: 16 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
<kms.version>1.40.0</kms.version>
<proto-kms.version>0.87.0</proto-kms.version>
<spotless-maven-plugin.version>2.12.1</spotless-maven-plugin.version>
<cloud-libraries-bom.version>24.2.0</cloud-libraries-bom.version>
</properties>

<dependencyManagement>
Expand All @@ -100,8 +101,15 @@
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>${cloud-libraries-bom.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
</dependencyManagement>

<dependencies>
<!-- Beam -->
Expand Down Expand Up @@ -478,6 +486,10 @@
<artifactId>proto</artifactId>
<version>${tensorflow.version}</version>
</dependency>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>google-cloud-secretmanager</artifactId>
</dependency>
</dependencies>

<build>
Expand Down Expand Up @@ -661,6 +673,9 @@
<ignoredUsedUndeclaredDependency>
org.apache.beam:beam-vendor-grpc-1_43_2
</ignoredUsedUndeclaredDependency>
<ignoredUsedUndeclaredDependency>
com.google.api.grpc:proto-google-cloud-secretmanager-v1
</ignoredUsedUndeclaredDependency>
</ignoredUsedUndeclaredDependencies>
<ignoredUnusedDeclaredDependencies>
<ignoredUnusedDeclaredDependency>org.threeten:threetenbp</ignoredUnusedDeclaredDependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
import com.google.cloud.teleport.templates.common.PubsubConverters.PubsubWriteDeadletterTopicOptions;
import com.google.cloud.teleport.templates.common.SplunkConverters;
import com.google.cloud.teleport.templates.common.SplunkConverters.SplunkOptions;
import com.google.cloud.teleport.util.KMSEncryptedNestedValueProvider;
import com.google.cloud.teleport.util.TokenNestedValueProvider;
import com.google.cloud.teleport.values.FailsafeElement;
import com.google.common.annotations.VisibleForTesting;
import com.google.gson.Gson;
Expand Down Expand Up @@ -245,7 +245,12 @@ public static PipelineResult run(PubSubToSplunkOptions options) {
.apply(
"WriteToSplunk",
SplunkIO.writeBuilder()
.withToken(maybeDecrypt(options.getToken(), options.getTokenKMSEncryptionKey()))
.withToken(
new TokenNestedValueProvider(
options.getTokenSecretId(),
options.getTokenKMSEncryptionKey(),
options.getToken(),
options.getTokenSource()))
.withUrl(options.getUrl())
.withBatchCount(options.getBatchCount())
.withParallelism(options.getParallelism())
Expand Down Expand Up @@ -367,19 +372,6 @@ public void processElement(ProcessContext context) {
}
}

/**
* Utility method to decrypt a Splunk HEC token.
*
* @param unencryptedToken The Splunk HEC token as a Base64 encoded {@link String} encrypted with
* a Cloud KMS Key.
* @param kmsKey The Cloud KMS Encryption Key to decrypt the Splunk HEC token.
* @return Decrypted Splunk HEC token.
*/
private static ValueProvider<String> maybeDecrypt(
ValueProvider<String> unencryptedToken, ValueProvider<String> kmsKey) {
return new KMSEncryptedNestedValueProvider(unencryptedToken, kmsKey);
}

/**
* Utility method that formats {@link org.apache.beam.sdk.io.gcp.pubsub.PubsubMessage} according
* to the model defined in {@link com.google.pubsub.v1.PubsubMessage}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,18 @@ public interface SplunkOptions extends PipelineOptions {

void setTokenKMSEncryptionKey(ValueProvider<String> keyName);

@Description(
"Secret Manager Secret ID for the token. Should be in the format "
+ "projects/{project}/secrets/{secret}/versions/{secret_version}")
ValueProvider<String> getTokenSecretId();

void setTokenSecretId(ValueProvider<String> secretId);

@Description("Source of the token. One of PLAINTEXT, KMS or SECRET_MANAGER.")
ValueProvider<String> getTokenSource();

void setTokenSource(ValueProvider<String> tokenSource);

@Description("Path to root CA in GCS, ex: gs://mybucket/somepath/rootCA.crt")
ValueProvider<String> getRootCaCertificatePath();

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (C) 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

package com.google.cloud.teleport.templates.common;

/** Supported sources for a Splunk Token. */
public enum SplunkTokenSource {
/** Token is passed as plaintext. */
PLAINTEXT,

/** Token to be decrypted using KMS. */
KMS,

/** Token is stored in Secret Manager. */
SECRET_MANAGER
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
* Copyright (C) 2021 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/

package com.google.cloud.teleport.util;

import com.google.cloud.secretmanager.v1.AccessSecretVersionResponse;
import com.google.cloud.secretmanager.v1.SecretManagerServiceClient;
import com.google.cloud.secretmanager.v1.SecretVersionName;
import java.io.IOException;
import java.io.Serializable;
import org.apache.beam.sdk.options.ValueProvider;
import org.apache.beam.sdk.transforms.SerializableFunction;

/**
* {@link SecretManagerValueProvider} class is a subclass of {@link ValueProvider} that takes in a
* {@link ValueProvider<String>} of the form
* projects/{project}/secrets/{secret}/versions/{secret_version} and returns the secret value in
* Secret Manager.
*/
public class SecretManagerValueProvider implements ValueProvider<String>, Serializable {

private transient volatile String cachedValue;
private final SerializableFunction<String, String> translator;
private final ValueProvider<String> secretVersion;

@Override
public String get() {
if (cachedValue == null) {
cachedValue = translator.apply(secretVersion.get());
}
return cachedValue;
}

@Override
public boolean isAccessible() {
return secretVersion.isAccessible();
}

private static class SecretTranslatorInput implements SerializableFunction<String, String> {

private SecretTranslatorInput() {}

public static SecretTranslatorInput of() {
return new SecretTranslatorInput();
}

@Override
public String apply(String secretVersion) {
SecretVersionName secretVersionName = parseSecretVersion(secretVersion);

try (SecretManagerServiceClient client = SecretManagerServiceClient.create()) {
AccessSecretVersionResponse response = client.accessSecretVersion(secretVersionName);
return response.getPayload().getData().toStringUtf8();
} catch (IOException e) {
throw new RuntimeException(e);
}
}

/**
* Parses a Secret Version and returns a {@link SecretVersionName}.
*
* @param secretVersion Secret Version of the form
* projects/{project}/secrets/{secret}/versions/{secret_version}
* @return {@link SecretVersionName}
*/
private SecretVersionName parseSecretVersion(String secretVersion) {
if (SecretVersionName.isParsableFrom(secretVersion)) {
return SecretVersionName.parse(secretVersion);
} else {
throw new IllegalArgumentException(
"Provided Secret must be in the form"
+ " projects/{project}/secrets/{secret}/versions/{secret_version}");
}
}
}

public SecretManagerValueProvider(ValueProvider<String> secretVersion) {
this.secretVersion = secretVersion;
this.translator = SecretTranslatorInput.of();
}
}
Loading

0 comments on commit 48d521e

Please sign in to comment.