Upgrade services from iceberg 1.2.0 to iceberg 1.5.2 (#179)

## Summary
Upgrade `services` from iceberg 1.2.0 to iceberg 1.5.2. `Integrations`,
`apps`, and `tables-test-fixtures` will remain iceberg 1.2.0.

Notable changes from iceberg 1.2.0 to iceberg 1.5.2 are not limited as
following:
- Add view support
- Add FileIO that supports ADLSv2 storage
- Support file and partition delete granularity
- Track partition statistics in TableMetadata
- Add last updated timestamp and snapshot ID to partitions metadata
table
- Added support for Spark 3.5 and removed support for Spark 3.2
- Add fast_forward procedure

## Changes
Added `openhouse.iceberg-conventions-1.2` to strictly pin the
dependencies version to be 1.2.0.

Excluded the iceberg libraries from the `tables` and
`tables-test-fixtures` module to remove the circular dependency.

**After the change, modules using iceberg 1.5.2 will be**:
- services
- cluster:storage
- internalcatalog
- htscatalog

**Modules using iceberg 1.2.0 will be**:
- apps
- integrations
- libs
- tables-test-fixtures

**Checkboxes**:
- [ ] Client-facing API Changes
- [ ] Internal API Changes
- [ ] Bug Fixes
- [ ] New Features
- [ ] Performance Improvements
- [ ] Code Style
- [x] Refactoring
- [ ] Documentation
- [ ] Tests

For all the boxes checked, please include additional details of the
changes made in this pull request.

## Testing Done
Comprehensive spark compatibility tests performed in the local cluster:
https://docs.google.com/document/d/1yXORH5ety5Gdr6Avsh7XEmIghiJ8wQrg0l4G4_9Soxo/edit#heading=h.h69v5xcbp1md

- [ ] Manually Tested on local docker setup. Please include commands
ran, and their output.
- [ ] Added new tests for the changes made.
- [x] Updated existing tests to reflect the changes made.
- [ ] No tests added or updated. Please explain why. If unsure, please
feel free to ask for help.
- [ ] Some other form of testing like staging or soak time in
production. Please explain.

For all the boxes checked, include a detailed description of the testing
done for the changes made in this pull request.

# Additional Information

- [ ] Breaking Changes
- [ ] Deprecations
- [ ] Large PR broken into smaller PRs, and PR plan linked in the
description.

For all the boxes checked, include additional details of the changes
made in this pull request.

apps/spark/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id 'openhouse.java-conventions'
     id 'openhouse.hadoop-conventions'
-    id 'openhouse.iceberg-conventions'
+    id 'openhouse.iceberg-conventions-1.2'
     id 'openhouse.maven-publish'
     id 'com.github.johnrengelman.shadow' version '7.1.2'
 }

buildSrc/src/main/groovy/openhouse.iceberg-conventions-1.2.gradle

@@ -0,0 +1,11 @@
+ext {
+  icebergVersion = '1.2.0'
+}
+
+dependencies {
+  implementation('org.apache.iceberg:iceberg-bundled-guava:' + icebergVersion + "!!")
+  implementation('org.apache.iceberg:iceberg-data:' + icebergVersion + "!!")
+  implementation('org.apache.iceberg:iceberg-core:' + icebergVersion + "!!")
+  implementation('org.apache.iceberg:iceberg-common:' + icebergVersion + "!!")
+  implementation('org.testcontainers:testcontainers:1.19.8')
+}

buildSrc/src/main/groovy/openhouse.iceberg-conventions.gradle → buildSrc/src/main/groovy/openhouse.iceberg-conventions-1.5.2.gradle

@@ -1,5 +1,5 @@
 ext {
-  icebergVersion = '1.2.0'
+  icebergVersion = '1.5.2'
 }
 
 dependencies {

cluster/storage/build.gradle

@@ -3,7 +3,7 @@ plugins {
   id 'openhouse.hadoop-conventions'
   id 'openhouse.iceberg-aws-conventions'
   id 'openhouse.iceberg-azure-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.maven-publish'
 }
 

iceberg/azure/build.gradle

@@ -1,6 +1,6 @@
 plugins {
   id 'openhouse.java-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.iceberg-azure-conventions'
   id 'openhouse.maven-publish'
 }

iceberg/openhouse/htscatalog/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id 'openhouse.springboot-conventions'
     id 'openhouse.hadoop-conventions'
-    id 'openhouse.iceberg-conventions'
+    id 'openhouse.iceberg-conventions-1.5.2'
     id 'openhouse.maven-publish'
 }
 

iceberg/openhouse/internalcatalog/build.gradle

@@ -1,7 +1,7 @@
 plugins {
   id 'openhouse.springboot-conventions'
   id 'openhouse.client-codegen-convention'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.iceberg-azure-conventions'
   id 'openhouse.hadoop-conventions'
   id 'openhouse.iceberg-aws-conventions'

iceberg/openhouse/internalcatalog/src/test/java/com/linkedin/openhouse/internal/catalog/OpenHouseInternalTableOperationsTest.java

@@ -114,8 +114,9 @@ void testDoCommitAppendSnapshotsInitialVersion() throws IOException {
 
       Map<String, String> updatedProperties = tblMetadataCaptor.getValue().properties();
       Assertions.assertEquals(
-          4,
-          updatedProperties.size()); /*location, lastModifiedTime, version and appended_snapshots*/
+          5,
+          updatedProperties
+              .size()); /*write.parquet.compression-codec, location, lastModifiedTime, version and appended_snapshots*/
       Assertions.assertEquals(
           "INITIAL_VERSION", updatedProperties.get(getCanonicalFieldName("tableVersion")));
       Assertions.assertEquals(
@@ -155,8 +156,9 @@ void testDoCommitAppendSnapshotsExistingVersion() throws IOException {
 
       Map<String, String> updatedProperties = tblMetadataCaptor.getValue().properties();
       Assertions.assertEquals(
-          4,
-          updatedProperties.size()); /*location, lastModifiedTime, version and deleted_snapshots*/
+          5,
+          updatedProperties
+              .size()); /*write.parquet.compression-codec, location, lastModifiedTime, version and deleted_snapshots*/
       Assertions.assertEquals(
           TEST_LOCATION, updatedProperties.get(getCanonicalFieldName("tableVersion")));
 
@@ -205,8 +207,9 @@ void testDoCommitAppendAndDeleteSnapshots() throws IOException {
 
       Map<String, String> updatedProperties = tblMetadataCaptor.getValue().properties();
       Assertions.assertEquals(
-          5,
-          updatedProperties.size()); /*location, lastModifiedTime, version and deleted_snapshots*/
+          6,
+          updatedProperties
+              .size()); /*write.parquet.compression-codec, location, lastModifiedTime, version, appended_snapshots and deleted_snapshots*/
       Assertions.assertEquals(
           TEST_LOCATION, updatedProperties.get(getCanonicalFieldName("tableVersion")));
 
@@ -259,8 +262,9 @@ void testDoCommitDeleteSnapshots() throws IOException {
 
       Map<String, String> updatedProperties = tblMetadataCaptor.getValue().properties();
       Assertions.assertEquals(
-          4,
-          updatedProperties.size()); /*location, lastModifiedTime, version and deleted_snapshots*/
+          5,
+          updatedProperties
+              .size()); /*write.parquet.compression-codec, location, lastModifiedTime, version and deleted_snapshots*/
       Assertions.assertEquals(
           TEST_LOCATION, updatedProperties.get(getCanonicalFieldName("tableVersion")));
 

iceberg/openhouse/internalcatalog/src/test/resources/wap_serialized_snapshots.json

@@ -1,5 +1,5 @@
 [
-  "{  \"snapshot-id\" : 1151407017102313399,  \"parent-snapshot-id\" : 1151407017102313398,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"operation\" : \"append\",    \"wap.id\" : \"wap1\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}",
-  "{  \"snapshot-id\" : 2151407017102313399,  \"parent-snapshot-id\" : 1151407017102313398,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"operation\" : \"append\",    \"wap.id\" : \"wap2\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}",
-  "{  \"snapshot-id\" : 3151407017102313399,  \"parent-snapshot-id\" : 1151407017102313399,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"source-snapshot-id\" : \"2151407017102313399\",    \"operation\" : \"append\",    \"published.wap.id\" : \"wap2\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}"
+  "{  \"snapshot-id\" : 1151407017102313399,  \"parent-snapshot-id\" : 1151407017102313398,  \"sequence-number\" : 1,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"operation\" : \"append\",    \"wap.id\" : \"wap1\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}",
+  "{  \"snapshot-id\" : 2151407017102313399,  \"parent-snapshot-id\" : 1151407017102313398,  \"sequence-number\" : 2,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"operation\" : \"append\",    \"wap.id\" : \"wap2\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}",
+  "{  \"snapshot-id\" : 3151407017102313399,  \"parent-snapshot-id\" : 1151407017102313399,  \"sequence-number\" : 3,  \"timestamp-ms\" : 1669126937912,  \"summary\" : {    \"source-snapshot-id\" : \"2151407017102313399\",    \"operation\" : \"append\",    \"published.wap.id\" : \"wap2\",    \"spark.app.id\" : \"local-1669126906634\",    \"added-data-files\" : \"1\",    \"added-records\" : \"1\",    \"added-files-size\" : \"673\",    \"changed-partition-count\" : \"1\",    \"total-records\" : \"1\",    \"total-files-size\" : \"673\",    \"total-data-files\" : \"1\",    \"total-delete-files\" : \"0\",    \"total-position-deletes\" : \"0\",    \"total-equality-deletes\" : \"0\"  },  \"manifest-list\" : \"/data/openhouse/db/test-7a9e8c95-1a62-4d29-9621-d8784047fc6b/metadata/snap-2151407017102313398-1-aa0dcbb9-707f-4f53-9df8-394bad8563f2.avro\",  \"schema-id\" : 0}"
 ]

libs/datalayout/build.gradle

@@ -1,7 +1,7 @@
 plugins {
     id 'openhouse.java-conventions'
     id 'openhouse.hadoop-conventions'
-    id 'openhouse.iceberg-conventions'
+    id 'openhouse.iceberg-conventions-1.2'
     id 'openhouse.maven-publish'
 }
 

services/common/build.gradle

@@ -1,6 +1,6 @@
 plugins {
   id 'openhouse.java-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.hadoop-conventions'
   id 'openhouse.maven-publish'
   id 'java-test-fixtures'

services/housetables/build.gradle

@@ -1,7 +1,7 @@
 plugins {
   id 'openhouse.springboot-ext-conventions'
   id 'openhouse.hadoop-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.maven-publish'
   /**
    * FIXME: Ideally, the below line are also defined in shared buildSrc. But raises following error:

services/jobs/build.gradle

@@ -1,7 +1,7 @@
 plugins {
   id 'openhouse.springboot-ext-conventions'
   id 'openhouse.hadoop-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.maven-publish'
   /**
    * FIXME: Ideally, the below line are also defined in shared buildSrc. But raises following error:

services/tables/build.gradle

@@ -3,7 +3,7 @@ plugins {
   id 'openhouse.hadoop-conventions'
   id 'openhouse.iceberg-aws-conventions'
   id 'openhouse.iceberg-azure-conventions'
-  id 'openhouse.iceberg-conventions'
+  id 'openhouse.iceberg-conventions-1.5.2'
   id 'openhouse.maven-publish'
   /**
    * FIXME: Ideally, the below line are also defined in shared buildSrc. But raises following error:
@@ -34,5 +34,7 @@ dependencies {
   testImplementation 'org.junit.jupiter:junit-jupiter-engine:' + junit_version
   testImplementation 'org.springframework.security:spring-security-test:5.7.3'
   testImplementation(testFixtures(project(':services:common')))
-  testImplementation project(':tables-test-fixtures_2.12')
+  testImplementation (project(':tables-test-fixtures_2.12')) {
+    exclude group: 'org.apache.iceberg'
+  }
 }

tables-test-fixtures/build.gradle

@@ -2,6 +2,7 @@ plugins {
   id 'openhouse.java-minimal-conventions'
   id 'com.github.johnrengelman.shadow' version '7.1.2'
   id 'openhouse.maven-publish'
+  id 'openhouse.iceberg-conventions-1.2'
 }
 
 import com.github.jengelman.gradle.plugins.shadow.transformers.PropertiesFileTransformer
@@ -16,7 +17,9 @@ configurations {
 dependencies {
   implementation 'org.junit.jupiter:junit-jupiter-engine:' + junit_version
   implementation 'org.springframework.boot:spring-boot-starter-test:' + spring_web_version
-  implementation(project(':services:tables'))
+  implementation ((project(':services:tables'))) {
+    exclude group: 'org.apache.iceberg'
+  }
   compileOnly 'org.springframework.boot:spring-boot-starter-tomcat:' + spring_web_version
   compileOnly('org.apache.spark:spark-sql_2.12:' + spark_version){
     // These classes are available from `client-codegen-convention.gradle`