Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

V2 migration glue #2052

Merged
merged 26 commits into from
Jul 22, 2024
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
d5f5b8f
update secrets manager to v2 in sdk
aimethed May 22, 2024
f889729
update ~half of connectors to use secrets manager v2
aimethed May 22, 2024
b717ea6
all others except ddb secrets v2
aimethed May 22, 2024
cd75a48
update ddb to secrets manager v2
aimethed May 24, 2024
fb591ee
remaining changes missed with initial passes
aimethed May 24, 2024
f11d7d0
updated glue to v2 in sdk
aimethed May 30, 2024
dfc3164
glue v2 for docdb and ddb
aimethed May 30, 2024
2616d10
glue v2 for hbase and elasticsearch
aimethed May 30, 2024
68a392c
glue v2 for gcs
aimethed May 30, 2024
4839562
glue v2 for neptune, redis, timestream
aimethed May 30, 2024
deeb559
glue v2 for kafka
aimethed May 30, 2024
40d76e4
glue v2 for msk
aimethed May 31, 2024
4e1fc93
removed hbase dependencies to shrink jar size
aimethed May 31, 2024
79d63b9
Merge branch 'v2-migration-secrets' into v2-migration-glue
aimethed May 31, 2024
04bd9f6
swap to apache http client for jar size reasons
aimethed May 31, 2024
37ff821
unused dependency remove hbase connector (#2021)
Trianz-Akshay Jun 14, 2024
35ee207
unused dependency remove bigquery connector (#2022)
Trianz-Akshay Jun 17, 2024
3ccf6e0
Merge branch 'master' into v2-migration-secrets
aimethed Jun 26, 2024
d35ea4b
Merge branch 'master' into v2-migration-glue
aimethed Jun 26, 2024
28b42dd
update ddb pom to use same v2 versions as sdk
aimethed Jun 27, 2024
ff03307
secrets manager v2 for clickhouse
aimethed Jun 27, 2024
c4fc0d6
Merge branch 'v2-migration-secrets' into v2-migration-glue
aimethed Jun 27, 2024
3be0aa1
attempt at v2 glue for merged kafka changes
aimethed Jun 27, 2024
fad93d7
Set Data format to GetSchemaVersionResponse object in test cases (#2057)
Jithendar12 Jun 28, 2024
720cc9e
use filters and streams
aimethed Jul 18, 2024
389516d
merge changes
aimethed Jul 22, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,8 @@
import com.amazonaws.athena.connector.lambda.security.EncryptionKeyFactory;
import com.amazonaws.athena.connectors.aws.cmdb.tables.TableProvider;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.apache.arrow.util.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -77,7 +77,7 @@ public AwsCmdbMetadataHandler(java.util.Map<String, String> configOptions)
protected AwsCmdbMetadataHandler(
TableProviderFactory tableProviderFactory,
EncryptionKeyFactory keyFactory,
AWSSecretsManager secretsManager,
SecretsManagerClient secretsManager,
AmazonAthena athena,
String spillBucket,
String spillPrefix,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import com.amazonaws.athena.connectors.aws.cmdb.tables.TableProvider;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.apache.arrow.util.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Map;

Expand Down Expand Up @@ -56,7 +56,7 @@ public AwsCmdbRecordHandler(java.util.Map<String, String> configOptions)
}

@VisibleForTesting
protected AwsCmdbRecordHandler(AmazonS3 amazonS3, AWSSecretsManager secretsManager, AmazonAthena athena, TableProviderFactory tableProviderFactory, java.util.Map<String, String> configOptions)
protected AwsCmdbRecordHandler(AmazonS3 amazonS3, SecretsManagerClient secretsManager, AmazonAthena athena, TableProviderFactory tableProviderFactory, java.util.Map<String, String> configOptions)
{
super(amazonS3, secretsManager, athena, SOURCE_TYPE, configOptions);
tableProviders = tableProviderFactory.getTableProviders();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@
import com.amazonaws.athena.connectors.aws.cmdb.tables.TableProvider;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.ArrayList;
import java.util.Collections;
Expand Down Expand Up @@ -98,7 +98,7 @@ public class AwsCmdbMetadataHandlerTest
private Block mockBlock;

@Mock
private AWSSecretsManager mockSecretsManager;
private SecretsManagerClient mockSecretsManager;

@Mock
private AmazonAthena mockAthena;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@
import com.amazonaws.athena.connectors.aws.cmdb.tables.TableProvider;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.MockitoJUnitRunner;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Collections;
import java.util.UUID;
Expand Down Expand Up @@ -77,7 +77,7 @@ public class AwsCmdbRecordHandlerTest
private TableProvider mockTableProvider;

@Mock
private AWSSecretsManager mockSecretsManager;
private SecretsManagerClient mockSecretsManager;

@Mock
private AmazonAthena mockAthena;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,14 @@
import com.amazonaws.athena.connectors.jdbc.manager.JDBCUtil;
import com.amazonaws.athena.connectors.mysql.MySqlMetadataHandler;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.pojo.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.sql.Connection;
import java.sql.PreparedStatement;
Expand Down Expand Up @@ -101,7 +101,7 @@ public ClickHouseMetadataHandler(DatabaseConnectionConfig databaseConnectionConf
@VisibleForTesting
protected ClickHouseMetadataHandler(
DatabaseConnectionConfig databaseConnectionConfig,
AWSSecretsManager secretsManager,
SecretsManagerClient secretsManager,
AmazonAthena athena,
JdbcConnectionFactory jdbcConnectionFactory,
java.util.Map<String, String> configOptions)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandler;
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandlerFactory;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.apache.arrow.util.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Map;

Expand Down Expand Up @@ -59,7 +59,7 @@ public ClickHouseMuxMetadataHandler(java.util.Map<String, String> configOptions)
}

@VisibleForTesting
protected ClickHouseMuxMetadataHandler(AWSSecretsManager secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
protected ClickHouseMuxMetadataHandler(SecretsManagerClient secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
Map<String, JdbcMetadataHandler> metadataHandlerMap, DatabaseConnectionConfig databaseConnectionConfig, java.util.Map<String, String> configOptions)
{
super(secretsManager, athena, jdbcConnectionFactory, metadataHandlerMap, databaseConnectionConfig, configOptions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcRecordHandlerFactory;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.google.common.annotations.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Map;

Expand Down Expand Up @@ -58,7 +58,7 @@ public ClickHouseMuxRecordHandler(java.util.Map<String, String> configOptions)
}

@VisibleForTesting
ClickHouseMuxRecordHandler(AmazonS3 amazonS3, AWSSecretsManager secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
ClickHouseMuxRecordHandler(AmazonS3 amazonS3, SecretsManagerClient secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
DatabaseConnectionConfig databaseConnectionConfig, Map<String, JdbcRecordHandler> recordHandlerMap, java.util.Map<String, String> configOptions)
{
super(amazonS3, secretsManager, athena, jdbcConnectionFactory, databaseConnectionConfig, recordHandlerMap, configOptions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,12 @@
import com.amazonaws.services.athena.AmazonAthenaClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder;
import com.google.common.annotations.VisibleForTesting;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.commons.lang3.Validate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.sql.Connection;
import java.sql.PreparedStatement;
Expand Down Expand Up @@ -77,12 +76,12 @@ public ClickHouseRecordHandler(DatabaseConnectionConfig databaseConnectionConfig

public ClickHouseRecordHandler(DatabaseConnectionConfig databaseConnectionConfig, JdbcConnectionFactory jdbcConnectionFactory, java.util.Map<String, String> configOptions)
{
this(databaseConnectionConfig, AmazonS3ClientBuilder.defaultClient(), AWSSecretsManagerClientBuilder.defaultClient(), AmazonAthenaClientBuilder.defaultClient(),
this(databaseConnectionConfig, AmazonS3ClientBuilder.defaultClient(), SecretsManagerClient.create(), AmazonAthenaClientBuilder.defaultClient(),
jdbcConnectionFactory, new MySqlQueryStringBuilder(MYSQL_QUOTE_CHARACTER, new MySqlFederationExpressionParser(MYSQL_QUOTE_CHARACTER)), configOptions);
}

@VisibleForTesting
ClickHouseRecordHandler(DatabaseConnectionConfig databaseConnectionConfig, final AmazonS3 amazonS3, final AWSSecretsManager secretsManager,
ClickHouseRecordHandler(DatabaseConnectionConfig databaseConnectionConfig, final AmazonS3 amazonS3, final SecretsManagerClient secretsManager,
final AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory, JdbcSplitQueryBuilder jdbcSplitQueryBuilder, java.util.Map<String, String> configOptions)
{
super(amazonS3, secretsManager, athena, databaseConnectionConfig, jdbcConnectionFactory, configOptions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,16 @@
import com.amazonaws.athena.connectors.jdbc.connection.JdbcConnectionFactory;
import com.amazonaws.athena.connectors.jdbc.connection.JdbcCredentialProvider;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.amazonaws.services.secretsmanager.model.GetSecretValueRequest;
import com.amazonaws.services.secretsmanager.model.GetSecretValueResult;
import net.jqwik.api.Table;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest;
import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse;

import java.sql.Connection;
import java.sql.PreparedStatement;
Expand Down Expand Up @@ -79,7 +79,7 @@ public class ClickHouseMetadataHandlerTest
private JdbcConnectionFactory jdbcConnectionFactory;
private Connection connection;
private FederatedIdentity federatedIdentity;
private AWSSecretsManager secretsManager;
private SecretsManagerClient secretsManager;
private AmazonAthena athena;
private BlockAllocator blockAllocator;

Expand All @@ -90,9 +90,9 @@ public void setup()
this.jdbcConnectionFactory = Mockito.mock(JdbcConnectionFactory.class);
this.connection = Mockito.mock(Connection.class, Mockito.RETURNS_DEEP_STUBS);
Mockito.when(this.jdbcConnectionFactory.getConnection(nullable(JdbcCredentialProvider.class))).thenReturn(this.connection);
this.secretsManager = Mockito.mock(AWSSecretsManager.class);
this.secretsManager = Mockito.mock(SecretsManagerClient.class);
this.athena = Mockito.mock(AmazonAthena.class);
Mockito.when(this.secretsManager.getSecretValue(Mockito.eq(new GetSecretValueRequest().withSecretId("testSecret")))).thenReturn(new GetSecretValueResult().withSecretString("{\"username\": \"testUser\", \"password\": \"testPassword\"}"));
Mockito.when(this.secretsManager.getSecretValue(Mockito.eq(GetSecretValueRequest.builder().secretId("testSecret").build()))).thenReturn(GetSecretValueResponse.builder().secretString("{\"username\": \"testUser\", \"password\": \"testPassword\"}").build());
this.metadataHandler = new ClickHouseMetadataHandler(databaseConnectionConfig, this.secretsManager, this.athena, this.jdbcConnectionFactory, com.google.common.collect.ImmutableMap.of());
this.federatedIdentity = Mockito.mock(FederatedIdentity.class);
this.blockAllocator = Mockito.mock(BlockAllocator.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@
import com.amazonaws.athena.connectors.jdbc.connection.JdbcConnectionFactory;
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandler;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Collections;
import java.util.Map;
Expand All @@ -49,7 +49,7 @@ public class ClickHouseMuxJdbcMetadataHandlerTest
private ClickHouseMetadataHandler metadataHandler;
private JdbcMetadataHandler jdbcMetadataHandler;
private BlockAllocator allocator;
private AWSSecretsManager secretsManager;
private SecretsManagerClient secretsManager;
private AmazonAthena athena;
private QueryStatusChecker queryStatusChecker;
private JdbcConnectionFactory jdbcConnectionFactory;
Expand All @@ -62,7 +62,7 @@ public void setup()
//Mockito.when(this.allocator.createBlock(nullable(Schema.class))).thenReturn(Mockito.mock(Block.class));
this.metadataHandler = Mockito.mock(ClickHouseMetadataHandler.class);
this.metadataHandlerMap = Collections.singletonMap("fakedatabase", this.metadataHandler);
this.secretsManager = Mockito.mock(AWSSecretsManager.class);
this.secretsManager = Mockito.mock(SecretsManagerClient.class);
this.athena = Mockito.mock(AmazonAthena.class);
this.queryStatusChecker = Mockito.mock(QueryStatusChecker.class);
this.jdbcConnectionFactory = Mockito.mock(JdbcConnectionFactory.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcRecordHandler;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.sql.Connection;
import java.sql.SQLException;
Expand All @@ -47,7 +47,7 @@ public class ClickHouseMuxJdbcRecordHandlerTest
private ClickHouseRecordHandler recordHandler;
private JdbcRecordHandler jdbcRecordHandler;
private AmazonS3 amazonS3;
private AWSSecretsManager secretsManager;
private SecretsManagerClient secretsManager;
private AmazonAthena athena;
private QueryStatusChecker queryStatusChecker;
private JdbcConnectionFactory jdbcConnectionFactory;
Expand All @@ -58,7 +58,7 @@ public void setup()
this.recordHandler = Mockito.mock(ClickHouseRecordHandler.class);
this.recordHandlerMap = Collections.singletonMap(ClickHouseConstants.NAME, this.recordHandler);
this.amazonS3 = Mockito.mock(AmazonS3.class);
this.secretsManager = Mockito.mock(AWSSecretsManager.class);
this.secretsManager = Mockito.mock(SecretsManagerClient.class);
this.athena = Mockito.mock(AmazonAthena.class);
this.queryStatusChecker = Mockito.mock(QueryStatusChecker.class);
this.jdbcConnectionFactory = Mockito.mock(JdbcConnectionFactory.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcArrowTypeConverter;
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandler;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
Expand All @@ -61,6 +60,7 @@
import org.apache.arrow.vector.types.pojo.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.sql.Connection;
import java.sql.DatabaseMetaData;
Expand Down Expand Up @@ -96,7 +96,7 @@ public HiveMetadataHandler(DatabaseConnectionConfig databaseConnectionConfig, ja
@VisibleForTesting
protected HiveMetadataHandler(
DatabaseConnectionConfig databaseConnectionConfiguration,
AWSSecretsManager secretManager,
SecretsManagerClient secretManager,
AmazonAthena athena,
JdbcConnectionFactory jdbcConnectionFactory,
java.util.Map<String, String> configOptions)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandler;
import com.amazonaws.athena.connectors.jdbc.manager.JdbcMetadataHandlerFactory;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import org.apache.arrow.util.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Map;

Expand Down Expand Up @@ -56,7 +56,7 @@ public HiveMuxMetadataHandler(java.util.Map<String, String> configOptions)
}

@VisibleForTesting
protected HiveMuxMetadataHandler(AWSSecretsManager secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
protected HiveMuxMetadataHandler(SecretsManagerClient secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
Map<String, JdbcMetadataHandler> metadataHandlerMap, DatabaseConnectionConfig databaseConnectionConfig, java.util.Map<String, String> configOptions)
{
super(secretsManager, athena, jdbcConnectionFactory, metadataHandlerMap, databaseConnectionConfig, configOptions);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import com.amazonaws.athena.connectors.jdbc.manager.JdbcRecordHandlerFactory;
import com.amazonaws.services.athena.AmazonAthena;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.secretsmanager.AWSSecretsManager;
import com.google.common.annotations.VisibleForTesting;
import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;

import java.util.Map;

Expand Down Expand Up @@ -57,7 +57,7 @@ public HiveMuxRecordHandler(java.util.Map<String, String> configOptions)
}

@VisibleForTesting
HiveMuxRecordHandler(AmazonS3 amazonS3, AWSSecretsManager secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
HiveMuxRecordHandler(AmazonS3 amazonS3, SecretsManagerClient secretsManager, AmazonAthena athena, JdbcConnectionFactory jdbcConnectionFactory,
DatabaseConnectionConfig databaseConnectionConfig, Map<String, JdbcRecordHandler> recordHandlerMap, java.util.Map<String, String> configOptions)
{
super(amazonS3, secretsManager, athena, jdbcConnectionFactory, databaseConnectionConfig, recordHandlerMap, configOptions);
Expand Down
Loading