Mcollado ci (apache#7)

* Add initial integration tests

* Add gradle workflow in github

* Fixed Dockerfile to create default-realm dir for sqlite

* Add docker-compose and Dockerfile for regtest

* Docker build in ci

* Fix context in docker-compose file

* Update regtest to work locally and in docker

* Fix dockerfile to run gradle build

* Add .keep file for output directory

.github/workflows/gradle.yml

@@ -0,0 +1,49 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle
+
+name: Java CI with Gradle
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: '21'
+          distribution: 'temurin'
+
+      # Configure Gradle for optimal use in GiHub Actions, including caching of downloaded dependencies.
+      # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0
+
+      - name: Build with Gradle Wrapper
+        working-directory: iceberg-rest-server
+        run: ./gradlew test
+
+    # NOTE: The Gradle Wrapper is the default and recommended way to run Gradle (https://docs.gradle.org/current/userguide/gradle_wrapper.html).
+    # If your project does not have the Gradle Wrapper configured, you can use the following configuration to run Gradle with a specified version.
+    #
+    # - name: Setup Gradle
+    #   uses: gradle/actions/setup-gradle@417ae3ccd767c252f5661f1ace9f835f9654f2b5 # v3.1.0
+    #   with:
+    #     gradle-version: '8.6'
+    #
+    # - name: Build with Gradle 8.6
+    #   run: gradle build

.github/workflows/regtest.yml

@@ -0,0 +1,18 @@
+name: Regression Tests
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Regression Test
+        run: docker compose up --exit-code-from regtest

.gitignore

@@ -2,3 +2,11 @@
 iceberg-rest-server/.gradle/*
 iceberg-rest-server/.idea/*
 iceberg-rest-server/build/*
+iceberg-rest-server/.java-version
+iceberg-rest-server/iceberg-rest-server.iml
+iceberg-rest-server/iceberg-rest-server.ipr
+iceberg-rest-server/iceberg-rest-server.iws
+iceberg-rest-server/logs/
+regtests/derby.log
+regtests/metastore_db
+regtests/output/

docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  pinnacle:
+    build:
+      context: ./iceberg-rest-server
+    ports:
+      - "8181:8181"
+  regtest:
+    build:
+      context: ./regtests
+      args:
+        PINNACLE_HOST: pinnacle
+    depends_on:
+      - pinnacle
+    volumes:
+      - local_output:/tmp/pinnacle-regtests/
+volumes:
+  local_output:
+    driver: local
+    driver_opts:
+      o: bind
+      type: none
+      device: ./regtests/output

Dockerfile → iceberg-rest-server/Dockerfile

@@ -2,20 +2,22 @@
 FROM openjdk:21 as build
 
 # Copy the REST catalog into the container
-COPY iceberg-rest-server /app
+COPY ./ /app
 
 # Set the working directory in the container, nuke any existing builds
 WORKDIR app
 CMD ["rm", "-rf", "build"]
 
 # Build the rest catalog
-CMD ["./gradlew", "build"]
+RUN ./gradlew build
 
 FROM openjdk:21
 WORKDIR /app
 COPY --from=build /app/build/libs/iceberg-rest-server-all.jar /app
 COPY --from=build /app/iceberg-rest-server.yml /app
 
+RUN mkdir -p /tmp/iceberg_rest_server_sqlitestate_basedir/default-realm
+
 # Expose the port that we run on (hardcoded as of now in `IcebergRestServerMain.java`)
 EXPOSE 8181
 

iceberg-rest-server/src/main/java/org/apache/iceberg/rest/IcebergRestApplication.java

@@ -68,7 +68,9 @@ public RealmContext resolveRealmContext(RESTCatalogAdapter.HTTPMethod method, St
                         return realmProperties::get;
                     }
                 })).addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true,"/*");
-        environment.jersey().register(new IcebergRestCatalogApi(new IcebergCatalogAdapter(new IcebergRestServerMain.CachingRealmCatalogFactory())));
+        IcebergRestServerMain.CachingRealmCatalogFactory catalogFactory =
+                new IcebergRestServerMain.CachingRealmCatalogFactory(configuration.getSqlLiteCatalogDirs());
+        environment.jersey().register(new IcebergRestCatalogApi(new IcebergCatalogAdapter(catalogFactory)));
         environment.jersey().register(new IcebergRestConfigurationApi(new IcebergRestConfigurationApiService() {
             @Override
             public Response getConfig(String warehouse, SecurityContext securityContext) {

iceberg-rest-server/src/main/java/org/apache/iceberg/rest/IcebergRestServerMain.java

@@ -14,25 +14,30 @@
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.servlet.ServletContextHandler;
-import org.eclipse.jetty.servlet.ServletHolder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 public class IcebergRestServerMain {
   private static final Logger LOG = LoggerFactory.getLogger(IcebergRestServerMain.class);
 
   private static final int SERVER_PORT = 8181;
-  private static final String METASTORE_STATE_BASEDIR =
+  private static final String DEFAULT_METASTORE_STATE_BASEDIR =
       "/tmp/iceberg_rest_server_sqlitestate_basedir/";
   private static final String WAREHOUSE_LOCATION_BASEDIR = "/tmp/iceberg_rest_server_warehouse_data/";
 
   static class CachingRealmCatalogFactory implements RealmCatalogFactory {
     private Map<String, Catalog> cachedCatalogs = new HashMap<>();
+    private final Map<String, String> catalogBaseDirs;
 
-    @Override
+      CachingRealmCatalogFactory(Map<String, String> catalogBaseDirs) {
+          this.catalogBaseDirs = catalogBaseDirs;
+      }
+
+      @Override
     public Catalog getOrCreateCatalog(RealmContext context, String catalogName) {
-      String realmName = context.getProperty("realm");
-      Catalog catalogInstance = cachedCatalogs.get(realmName);
+        String realm = context.getProperty("realm");
+        String catalogKey = realm + "/" + catalogName;
+      Catalog catalogInstance = cachedCatalogs.get(catalogKey);
       if (catalogInstance == null) {
         Map<String, String> catalogProperties = new HashMap<>();
         catalogProperties.put(
@@ -41,25 +46,26 @@ public Catalog getOrCreateCatalog(RealmContext context, String catalogName) {
 
         // TODO: Do sanitization in case this ever runs in an exposed prod environment to avoid
         // injection attacks.
+        String baseDir = catalogBaseDirs.getOrDefault(realm, DEFAULT_METASTORE_STATE_BASEDIR);
         catalogProperties.put(
-            CatalogProperties.URI, "jdbc:sqlite:file:" + METASTORE_STATE_BASEDIR + realmName);
+            CatalogProperties.URI, "jdbc:sqlite:file:" + baseDir + "/" + catalogKey);
 
         // TODO: Derive warehouse location from ream configs.
         catalogProperties.put(
-            CatalogProperties.WAREHOUSE_LOCATION, WAREHOUSE_LOCATION_BASEDIR + realmName);
+            CatalogProperties.WAREHOUSE_LOCATION, WAREHOUSE_LOCATION_BASEDIR + catalogKey);
 
         catalogInstance = CatalogUtil.buildIcebergCatalog(
-        "catalog_" + realmName, catalogProperties, new Configuration());
-        cachedCatalogs.put(realmName, catalogInstance);
+        "catalog_" + catalogKey, catalogProperties, new Configuration());
+        cachedCatalogs.put(catalogKey, catalogInstance);
       }
       return catalogInstance;
     }
   }
 
   public static void main(String[] args) throws Exception {
     // Ensure parent directories of metastore-state base directory exists.
-    LOG.info("Creating metastore state directory: " + METASTORE_STATE_BASEDIR);
-    Path result = Files.createDirectories(FileSystems.getDefault().getPath(METASTORE_STATE_BASEDIR));
+    LOG.info("Creating metastore state directory: " + DEFAULT_METASTORE_STATE_BASEDIR);
+    Path result = Files.createDirectories(FileSystems.getDefault().getPath(DEFAULT_METASTORE_STATE_BASEDIR));
 
     RESTCatalogAdapter adapter = new RealmContextDelegatingRESTCatalogAdapter(
         new RealmContextResolver() {
@@ -103,7 +109,7 @@ public String getProperty(String key) {
             };
           }
         },
-        new CachingRealmCatalogFactory());
+        new CachingRealmCatalogFactory(Map.of()));
     RESTCatalogServlet servlet = new RESTCatalogServlet(adapter);
 
     ServletContextHandler context = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);

iceberg-rest-server/src/main/java/org/apache/iceberg/rest/config/IcebergRestApplicationConfig.java

@@ -2,5 +2,17 @@
 
 import io.dropwizard.core.Configuration;
 
+import java.util.HashMap;
+import java.util.Map;
+
 public class IcebergRestApplicationConfig extends Configuration {
+    private Map<String, String> sqlLiteCatalogDirs = new HashMap<>();
+
+    public Map<String, String> getSqlLiteCatalogDirs() {
+        return sqlLiteCatalogDirs;
+    }
+
+    public void setSqlLiteCatalogDirs(Map<String, String> sqlLiteCatalogDirs) {
+        this.sqlLiteCatalogDirs = sqlLiteCatalogDirs;
+    }
 }

iceberg-rest-server/src/test/java/org/apache/iceberg/rest/IcebergRestApplicationIntegrationTest.java

@@ -0,0 +1,146 @@
+package org.apache.iceberg.rest;
+
+import io.dropwizard.testing.ResourceHelpers;
+import io.dropwizard.testing.junit5.DropwizardAppExtension;
+import io.dropwizard.testing.junit5.DropwizardExtensionsSupport;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SessionCatalog;
+import org.apache.iceberg.exceptions.NoSuchNamespaceException;
+import org.apache.iceberg.rest.config.IcebergRestApplicationConfig;
+import org.assertj.core.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
+
+@ExtendWith(DropwizardExtensionsSupport.class)
+public class IcebergRestApplicationIntegrationTest {
+    private static DropwizardAppExtension<IcebergRestApplicationConfig> EXT =
+            new DropwizardAppExtension<>(
+                    IcebergRestApplication.class,
+            ResourceHelpers.resourceFilePath("iceberg-rest-server-integrationtest.yml")
+    );
+
+    @BeforeAll
+    public static void setup() throws IOException {
+        Path testDir = Path.of("build/test_data/iceberg/default-realm");
+        if (Files.exists(testDir)) {
+            if (Files.isDirectory(testDir)) {
+                Files.walk(testDir)
+                        .sorted(Comparator.reverseOrder())
+                        .forEach(path -> {
+                            try {
+                                Files.delete(path);
+                            } catch (IOException e) {
+                                throw new RuntimeException(e);
+                            }
+                        });
+
+            } else {
+                Files.delete(testDir);
+            }
+        }
+        Files.createDirectories(testDir);
+    }
+
+    private static RESTSessionCatalog newSessionCatalog(String catalog) {
+        RESTSessionCatalog sessionCatalog = new RESTSessionCatalog();
+        sessionCatalog.initialize("snowflake", Map.of(
+                "uri", "http://localhost:" + EXT.getLocalPort() + "/api/catalog",
+                "prefix", catalog
+        ));
+        return sessionCatalog;
+    }
+
+    @Test
+    public void testIcebergListNamespaces() throws IOException {
+        try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListNamespaces")) {
+            SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+            List<Namespace> namespaces = sessionCatalog.listNamespaces(sessionContext);
+            assertThat(namespaces)
+                    .isNotNull()
+                    .isEmpty();
+        }
+    }
+
+    @Test
+    public void testIcebergListNamespacesNotFound() throws IOException {
+        try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListNamespacesNotFound")) {
+            SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+            try {
+                sessionCatalog.listNamespaces(sessionContext, Namespace.of("whoops"));
+                fail("Expected exception to be thrown");
+            } catch (NoSuchNamespaceException e) {
+                // we expect this!
+                Assertions.assertThat(e).isNotNull();
+            } catch (Exception e) {
+                fail("Unexpected exception", e);
+            }
+        }
+    }
+
+    @Test
+    public void testIcebergListNamespacesNestedNotFound() throws IOException {
+        try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListNamespacesNestedNotFound")) {
+            SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+            Namespace topLevelNamespace = Namespace.of("top_level");
+            sessionCatalog.createNamespace(sessionContext, topLevelNamespace);
+            sessionCatalog.loadNamespaceMetadata(sessionContext, Namespace.of("top_level"));
+            try {
+                sessionCatalog.listNamespaces(sessionContext, Namespace.of("top_level", "whoops"));
+                fail("Expected exception to be thrown");
+            } catch (NoSuchNamespaceException e) {
+                // we expect this!
+                Assertions.assertThat(e).isNotNull();
+            } catch (Exception e) {
+                fail("Unexpected exception", e);
+            }
+        }
+    }
+
+    @Test
+    public void testIcebergListTablesNamespaceNotFound() throws IOException {
+        try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergListTablesNamespaceNotFound")) {
+            SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+            try {
+                sessionCatalog.listTables(sessionContext, Namespace.of("whoops"));
+                fail("Expected exception to be thrown");
+            } catch (NoSuchNamespaceException e) {
+                // we expect this!
+                Assertions.assertThat(e).isNotNull();
+            } catch (Exception e) {
+                fail("Unexpected exception", e);
+            }
+        }
+    }
+
+    @Test
+    public void testIcebergCreateNamespace() throws IOException {
+        try (RESTSessionCatalog sessionCatalog = newSessionCatalog("testIcebergCreateNamespace")) {
+            SessionCatalog.SessionContext sessionContext = SessionCatalog.SessionContext.createEmpty();
+            Namespace topLevelNamespace = Namespace.of("top_level");
+            sessionCatalog.createNamespace(sessionContext, topLevelNamespace);
+            List<Namespace> namespaces = sessionCatalog.listNamespaces(sessionContext);
+            assertThat(namespaces)
+                    .isNotNull()
+                    .hasSize(1)
+                    .containsExactly(topLevelNamespace);
+            Namespace nestedNamespace = Namespace.of("top_level", "second_level");
+            sessionCatalog.createNamespace(sessionContext, nestedNamespace);
+            namespaces = sessionCatalog.listNamespaces(sessionContext, topLevelNamespace);
+            assertThat(namespaces)
+                    .isNotNull()
+                    .hasSize(1)
+                    .containsExactly(nestedNamespace);
+        }
+    }
+}