Skip to content

Commit

Permalink
Refactor PolarisRestCatalogViewIntegrationTest into per cloud provide…
Browse files Browse the repository at this point in the history
…r tests (apache#410)

* Update to be 1 test

* Spotless

* Go back to multiple files
  • Loading branch information
andrew4699 authored Oct 30, 2024
1 parent 041a8ff commit 121ae9a
Show file tree
Hide file tree
Showing 5 changed files with 208 additions and 26 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.service.catalog;

import java.util.List;
import java.util.Optional;
import java.util.stream.Stream;
import org.apache.polaris.core.admin.model.AwsStorageConfigInfo;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.assertj.core.util.Strings;

/** Runs PolarisRestCatalogViewIntegrationTest on AWS. */
public class PolarisRestCatalogViewAwsIntegrationTest
extends PolarisRestCatalogViewIntegrationTest {
public static final String ROLE_ARN =
Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN")) // Backward compatibility
.orElse(System.getenv("INTEGRATION_TEST_S3_ROLE_ARN"));
public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_S3_PATH");

@Override
protected StorageConfigInfo getStorageConfigInfo() {
return AwsStorageConfigInfo.builder()
.setRoleArn(ROLE_ARN)
.setStorageType(StorageConfigInfo.StorageTypeEnum.S3)
.setAllowedLocations(List.of(BASE_LOCATION))
.build();
}

@Override
protected boolean shouldSkip() {
return Stream.of(BASE_LOCATION, ROLE_ARN).anyMatch(Strings::isNullOrEmpty);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.service.catalog;

import java.util.List;
import java.util.stream.Stream;
import org.apache.polaris.core.admin.model.AzureStorageConfigInfo;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.assertj.core.util.Strings;

/** Runs PolarisRestCatalogViewIntegrationTest on Azure. */
public class PolarisRestCatalogViewAzureIntegrationTest
extends PolarisRestCatalogViewIntegrationTest {
public static final String TENANT_ID = System.getenv("INTEGRATION_TEST_AZURE_TENANT_ID");
public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_AZURE_PATH");

@Override
protected StorageConfigInfo getStorageConfigInfo() {
return AzureStorageConfigInfo.builder()
.setTenantId(TENANT_ID)
.setStorageType(StorageConfigInfo.StorageTypeEnum.AZURE)
.setAllowedLocations(List.of(BASE_LOCATION))
.build();
}

@Override
protected boolean shouldSkip() {
return Stream.of(BASE_LOCATION, TENANT_ID).anyMatch(Strings::isNullOrEmpty);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.service.catalog;

import java.util.List;
import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
import org.apache.polaris.core.admin.model.StorageConfigInfo;

/** Runs PolarisRestCatalogViewIntegrationTest on the local filesystem. */
public class PolarisRestCatalogViewFileIntegrationTest
extends PolarisRestCatalogViewIntegrationTest {
public static final String BASE_LOCATION = "file:///tmp/buckets/my-bucket";

@Override
protected StorageConfigInfo getStorageConfigInfo() {
return FileStorageConfigInfo.builder()
.setStorageType(StorageConfigInfo.StorageTypeEnum.FILE)
.setAllowedLocations(List.of(BASE_LOCATION))
.build();
}

@Override
protected boolean shouldSkip() {
return false;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.polaris.service.catalog;

import java.util.List;
import java.util.stream.Stream;
import org.apache.polaris.core.admin.model.GcpStorageConfigInfo;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.assertj.core.util.Strings;

/** Runs PolarisRestCatalogViewIntegrationTest on GCP. */
public class PolarisRestCatalogViewGcpIntegrationTest
extends PolarisRestCatalogViewIntegrationTest {
public static final String SERVICE_ACCOUNT =
System.getenv("INTEGRATION_TEST_GCS_SERVICE_ACCOUNT");
public static final String BASE_LOCATION = System.getenv("INTEGRATION_TEST_GCS_PATH");

@Override
protected StorageConfigInfo getStorageConfigInfo() {
return GcpStorageConfigInfo.builder()
.setGcsServiceAccount(SERVICE_ACCOUNT)
.setStorageType(StorageConfigInfo.StorageTypeEnum.GCS)
.setAllowedLocations(List.of(BASE_LOCATION))
.build();
}

@Override
protected boolean shouldSkip() {
return Stream.of(BASE_LOCATION, SERVICE_ACCOUNT).anyMatch(Strings::isNullOrEmpty);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,11 @@
import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
import jakarta.ws.rs.core.Response;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.apache.iceberg.rest.RESTCatalog;
import org.apache.iceberg.view.ViewCatalogTests;
import org.apache.polaris.core.PolarisConfiguration;
import org.apache.polaris.core.admin.model.AwsStorageConfigInfo;
import org.apache.polaris.core.admin.model.Catalog;
import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
import org.apache.polaris.core.admin.model.PolarisCatalog;
import org.apache.polaris.core.admin.model.StorageConfigInfo;
import org.apache.polaris.core.entity.CatalogEntity;
Expand All @@ -45,6 +41,7 @@
import org.apache.polaris.service.test.PolarisRealm;
import org.apache.polaris.service.test.SnowmanCredentialsExtension;
import org.apache.polaris.service.test.SnowmanCredentialsExtension.SnowmanCredentials;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
Expand All @@ -59,13 +56,7 @@
PolarisConnectionExtension.class,
SnowmanCredentialsExtension.class
})
public class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests<RESTCatalog> {
public static final String TEST_ROLE_ARN =
Optional.ofNullable(System.getenv("INTEGRATION_TEST_ROLE_ARN"))
.orElse("arn:aws:iam::123456789012:role/my-role");
public static final String S3_BUCKET_BASE =
Optional.ofNullable(System.getenv("INTEGRATION_TEST_S3_PATH"))
.orElse("file:///tmp/buckets/my-bucket");
public abstract class PolarisRestCatalogViewIntegrationTest extends ViewCatalogTests<RESTCatalog> {
private static final DropwizardAppExtension<PolarisApplicationConfig> EXT =
new DropwizardAppExtension<>(
PolarisApplication.class,
Expand All @@ -90,6 +81,9 @@ public void before(
PolarisToken adminToken,
SnowmanCredentials snowmanCredentials,
@PolarisRealm String realm) {

Assumptions.assumeFalse(shouldSkip());

String userToken = adminToken.token();
testInfo
.getTestMethod()
Expand All @@ -114,17 +108,14 @@ public void before(
}
}

AwsStorageConfigInfo awsConfigModel =
AwsStorageConfigInfo.builder()
.setRoleArn(TEST_ROLE_ARN)
.setExternalId("externalId")
.setUserArn("userArn")
.setStorageType(StorageConfigInfo.StorageTypeEnum.S3)
.setAllowedLocations(List.of("s3://my-old-bucket/path/to/data"))
.build();
StorageConfigInfo storageConfig = getStorageConfigInfo();
String defaultBaseLocation =
storageConfig.getAllowedLocations().getFirst()
+ "/"
+ System.getenv("USER")
+ "/path/to/data";
org.apache.polaris.core.admin.model.CatalogProperties props =
org.apache.polaris.core.admin.model.CatalogProperties.builder(
S3_BUCKET_BASE + "/" + System.getenv("USER") + "/path/to/data")
org.apache.polaris.core.admin.model.CatalogProperties.builder(defaultBaseLocation)
.addProperty(
CatalogEntity.REPLACE_NEW_LOCATION_PREFIX_WITH_CATALOG_DEFAULT_KEY,
"file:")
Expand All @@ -140,18 +131,25 @@ public void before(
.setType(Catalog.TypeEnum.INTERNAL)
.setName(catalogName)
.setProperties(props)
.setStorageConfigInfo(
S3_BUCKET_BASE.startsWith("file:")
? new FileStorageConfigInfo(
StorageConfigInfo.StorageTypeEnum.FILE, List.of("file://"))
: awsConfigModel)
.setStorageConfigInfo(storageConfig)
.build();
restCatalog =
TestUtil.createSnowmanManagedCatalog(
EXT, adminToken, snowmanCredentials, realm, catalog, Map.of());
});
}

/**
* @return The catalog's storage config.
*/
protected abstract StorageConfigInfo getStorageConfigInfo();

/**
* @return Whether the tests should be skipped, for example due to environment variables not being
* specified.
*/
protected abstract boolean shouldSkip();

@Override
protected RESTCatalog catalog() {
return restCatalog;
Expand Down

0 comments on commit 121ae9a

Please sign in to comment.