Skip to content

[test] Remove the S3 dependency from paimon-hive and paimon-spark #7279

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JingsongLi merged 1 commit into from
Feb 13, 2026
Merged

[test] Remove the S3 dependency from paimon-hive and paimon-spark #7279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions ...hive-connector-common/src/test/java/org/apache/paimon/hive/FlinkGenericCatalogITCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@

import org.apache.paimon.flink.FlinkGenericCatalog;
import org.apache.paimon.flink.FlinkGenericCatalogFactory;
import org.apache.paimon.hive.annotation.Minio;
import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.s3.MinioTestContainer;

import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList;

Expand Down Expand Up @@ -62,8 +60,6 @@ public class FlinkGenericCatalogITCase extends AbstractTestBaseJUnit4 {
@HiveSQL(files = {})
protected static HiveShell hiveShell;

@Minio private static MinioTestContainer minioTestContainer;

private static HiveCatalog createHiveCatalog(HiveConf hiveConf) {
return new HiveCatalog(
"testcatalog", null, hiveConf, HiveShimLoader.getHiveVersion(), true);
Expand Down
27 changes: 6 additions & 21 deletions ...mon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveCatalogITCaseBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,10 @@
import org.apache.paimon.flink.FlinkCatalog;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.hive.annotation.Minio;
import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.operation.Lock;
import org.apache.paimon.options.Options;
import org.apache.paimon.privilege.NoPrivilegeException;
import org.apache.paimon.s3.MinioTestContainer;
import org.apache.paimon.table.CatalogEnvironment;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.Table;
Expand Down Expand Up @@ -72,7 +70,6 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
Expand All @@ -98,15 +95,9 @@ public abstract class HiveCatalogITCaseBase {
@HiveSQL(files = {})
protected static HiveShell hiveShell;

@Minio private static MinioTestContainer minioTestContainer;

private void before(boolean locationInProperties) throws Exception {
this.locationInProperties = locationInProperties;
if (locationInProperties) {
path = minioTestContainer.getS3UriForDefaultBucket() + "/" + UUID.randomUUID();
} else {
path = folder.newFolder().toURI().toString();
}
path = folder.newFolder().toURI().toString();
registerHiveCatalog("my_hive", new HashMap<>());

tEnv.executeSql("USE CATALOG my_hive").await();
Expand All @@ -130,9 +121,6 @@ private void registerHiveCatalog(String catalogName, Map<String, String> catalog
catalogProperties.put("lock.enabled", "true");
catalogProperties.put("location-in-properties", String.valueOf(locationInProperties));
catalogProperties.put("warehouse", path);
if (locationInProperties) {
catalogProperties.putAll(minioTestContainer.getS3ConfigOptions());
}

Options catalogOptions = new Options(catalogProperties);
CatalogContext catalogContext = CatalogContext.create(catalogOptions);
Expand Down Expand Up @@ -193,23 +181,20 @@ public void evaluate() throws Throwable {
};

@Test
public void testDbLocation() {
String dbLocation = minioTestContainer.getS3UriForDefaultBucket() + "/" + UUID.randomUUID();
public void testDbLocation() throws Exception {
String dbLocation = folder.newFolder().toURI().toString();
Catalog catalog =
((FlinkCatalog) tEnv.getCatalog(tEnv.getCurrentCatalog()).get()).catalog();
Map<String, String> properties = new HashMap<>();
properties.put("location", dbLocation);

assertThatThrownBy(() -> catalog.createDatabase("location_test_db", false, properties))
.hasMessageContaining(
"Could not find a file io implementation for scheme 's3' in the classpath.");
catalog.createDatabase("location_test_db", false, properties);
}

@Test
@LocationInProperties
public void testDbLocationWithMetastoreLocationInProperties()
throws Catalog.DatabaseAlreadyExistException, Catalog.DatabaseNotExistException {
String dbLocation = minioTestContainer.getS3UriForDefaultBucket() + "/" + UUID.randomUUID();
public void testDbLocationWithMetastoreLocationInProperties() throws Exception {
String dbLocation = folder.newFolder().toURI().toString();
Catalog catalog =
((FlinkCatalog) tEnv.getCatalog(tEnv.getCurrentCatalog()).get()).catalog();
Map<String, String> properties = new HashMap<>();
Expand Down
34 changes: 2 additions & 32 deletions ...e/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,9 @@
import org.apache.paimon.catalog.Identifier;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.fs.Path;
import org.apache.paimon.hive.annotation.Minio;
import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner;
import org.apache.paimon.options.CatalogOptions;
import org.apache.paimon.options.Options;
import org.apache.paimon.s3.MinioTestContainer;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaManager;
import org.apache.paimon.schema.TableSchema;
Expand Down Expand Up @@ -55,22 +53,18 @@
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;

import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;

/** Test for specify location. */
@RunWith(PaimonEmbeddedHiveRunner.class)
public class HiveLocationTest {
@HiveSQL(files = {})
private static HiveShell hiveShell;

@Minio private static MinioTestContainer minioTestContainer;

public static final String HIVE_CONF = "/hive-conf";

private HiveCatalog catalog;
Expand All @@ -96,11 +90,6 @@ public void before() throws IOException {
options.set(HiveCatalogOptions.HIVE_CONF_DIR, hiveShell.getBaseDir() + HIVE_CONF);
options.set(HiveCatalogOptions.LOCATION_IN_PROPERTIES, true);

for (Map.Entry<String, String> stringStringEntry :
minioTestContainer.getS3ConfigOptions().entrySet()) {
options.set(stringStringEntry.getKey(), stringStringEntry.getValue());
}

// create CatalogContext using the options
catalogContext = CatalogContext.create(options);

Expand All @@ -112,14 +101,6 @@ public void before() throws IOException {
catalog = (HiveCatalog) hiveCatalogFactory.create(catalogContext);

hmsClient = catalog.getHmsClient();

String setTemplate = "SET paimon.%s=%s";
minioTestContainer
.getS3ConfigOptions()
.forEach(
(k, v) -> {
hiveShell.execute(String.format(setTemplate, k, v));
});
}

private static FileIO getFileIO(CatalogContext catalogContext, Path warehouse) {
Expand Down Expand Up @@ -204,24 +185,13 @@ public void testCatalogTableLocation() throws Exception {
@Test
public void testExternTableLocation() throws Exception {

String path = minioTestContainer.getS3UriForDefaultBucket() + "/" + UUID.randomUUID();
String path = hiveShell.getBaseDir().toAbsolutePath().toString() + "/" + UUID.randomUUID();

Options conf = new Options();
conf.set(CatalogOptions.WAREHOUSE, path);

for (Map.Entry<String, String> stringStringEntry :
minioTestContainer.getS3ConfigOptions().entrySet()) {
conf.set(stringStringEntry.getKey(), stringStringEntry.getValue());
}

RowType rowType = RowType.of(new DataType[] {DataTypes.INT()}, new String[] {"aaa"});
// create table with location field
assertThatThrownBy(
() ->
createTableWithStorageLocation(
path, rowType, "test_extern_table", conf, true))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining("No FileSystem for scheme: s3");
createTableWithStorageLocation(path, rowType, "test_extern_table", conf, true);

// create table with location in table properties
Set<String> tableForTest = Sets.newHashSet("test_extern_table1", "hive_inner_table1");
Expand Down
40 changes: 0 additions & 40 deletions ...onnector-common/src/test/java/org/apache/paimon/hive/runner/PaimonEmbeddedHiveRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.apache.paimon.hive.runner;

import org.apache.paimon.hive.annotation.Minio;
import org.apache.paimon.s3.MinioTestContainer;
import org.apache.paimon.utils.Preconditions;

import org.apache.paimon.shade.guava30.com.google.common.io.Resources;
Expand Down Expand Up @@ -110,24 +108,6 @@ protected void after() {
}
};

ExternalResource minio =
new ExternalResource() {
public MinioTestContainer minioTestContainer;

@Override
protected void before() {
minioTestContainer = setMinioTestContainer(getTestClass().getJavaClass());
}

@Override
protected void after() {
if (minioTestContainer != null && minioTestContainer.isRunning()) {
minioTestContainer.close();
}
}
};

rules.add(minio);
rules.add(hiveShell);
return rules;
}
Expand Down Expand Up @@ -369,26 +349,6 @@ private void loadAnnotatedProperties(Class testClass, HiveShellBuilder workFlowB
}
}

private MinioTestContainer setMinioTestContainer(final Class testClass) {
Set<Field> allFields = ReflectionUtils.getAllFields(testClass, withAnnotation(Minio.class));

Preconditions.checkState(
allFields.size() <= 1,
"At most one field of type MinioTestContainer should to be annotated with @MinIO");
MinioTestContainer minioTestContainer = null;
if (!allFields.isEmpty()) {
minioTestContainer = new MinioTestContainer();
minioTestContainer.start();

Field field = allFields.iterator().next();
Preconditions.checkState(
ReflectionUtils.isOfType(field, MinioTestContainer.class),
"Field annotated with @MinIO should be of type MinioTestContainer");
ReflectionUtils.setStaticField(testClass, field.getName(), minioTestContainer);
}
return minioTestContainer;
}

/**
* Used as a handle for the HiveShell field in the test case so that we may set it once the
* HiveShell has been instantiated.
Expand Down
30 changes: 0 additions & 30 deletions paimon-hive/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@ under the License.
<tez.version>0.10.0</tez.version>
<hiverunner.version>5.5.0</hiverunner.version>
<reflections.version>0.9.8</reflections.version>
<aws.version>1.12.319</aws.version>
<iceberg.flink.version>1.19</iceberg.flink.version>
<iceberg.flink.dropwizard.version>1.19.0</iceberg.flink.dropwizard.version>
</properties>
Expand Down Expand Up @@ -96,35 +95,6 @@ under the License.
</exclusions>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-s3</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-s3</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-core</artifactId>
<version>${aws.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>${aws.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-test-utils</artifactId>
Expand Down
Loading
Loading