Merge remote-tracking branch 'origin/main' into s3fileio-cross-region…

…-bucket-access
apache · Oct 5, 2024 · ef5af04 · ef5af04
2 parents 4303eeb + 8190ce7
commit ef5af04
Show file tree

Hide file tree

Showing 2,684 changed files with 175,325 additions and 104,004 deletions.
diff --git a/.asf.yaml b/.asf.yaml
@@ -35,6 +35,9 @@ github:
 
   protected_branches:
     main:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+
       required_linear_history: true
 
   features:
@@ -46,12 +49,12 @@ github:
     - jun-he
     - marton-bod
     - samarthjain
-    - findepi
     - SreeramGarlapati
     - samredai
     - gaborkaszab
     - bitsondatadev
     - ajantha-bhat
+    - jbonofre
   ghp_branch: gh-pages
   ghp_path: /
 

diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
@@ -88,6 +88,14 @@
         <property name="format" value="Preconditions\.checkArgument\([^;]+%d[^;]+\);"/>
         <property name="message" value="Preconditions.checkArgument does not support %d. use %s instead"/>
     </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="^\s*import\s+static\s+(?!\Qorg.assertj.core.api.Assertions.\E).*\.assertThatThrownBy;"/>
+        <property name="message" value="assertThatThrownBy() should be statically imported from org.assertj.core.api.Assertions"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="^\s*import\s+\Qorg.assertj.core.api.Assertions;\E" />
+        <property name="message" value="org.assertj.core.api.Assertions should only be used with static imports" />
+    </module>
     <module name="SuppressionFilter"> <!-- baseline-gradle: README.md -->
         <property name="file" value="${config_loc}/checkstyle-suppressions.xml"/>
     </module>
@@ -145,11 +153,12 @@
                 org.apache.spark.sql.functions.*,
                 org.apache.spark.sql.connector.iceberg.write.RowLevelOperation.Command.*,
                 org.apache.spark.sql.connector.write.RowLevelOperation.Command.*,
-                org.junit.Assert.*"/>
+                org.junit.Assert.*,
+                org.assertj.core.api.Assertions.*,
+                org.assertj.core.api.Assumptions.*"/>
         </module>
         <module name="ClassTypeParameterName"> <!-- Java Style Guide: Type variable names -->
             <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
-            <message key="name.invalidPattern" value="Class type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="CovariantEquals"/> <!-- Java Coding Guidelines: Override ``Object#equals`` consistently -->
         <module name="DefaultComesLast"/> <!-- Java Style Guide: The default case is present -->
@@ -271,12 +280,13 @@
         </module>
         <module name="InnerAssignment"/> <!-- Java Coding Guidelines: Inner assignments: Not used -->
         <module name="MemberName"> <!-- Java Style Guide: Non-constant field names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
-            <message key="name.invalidPattern" value="Member name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
+        </module>
+        <module name="ConstantName">
+            <property name="format" value="^[A-Z][A-Z0-9]*+(_[A-Z0-9]++)*+$"/>
         </module>
         <module name="MethodName"> <!-- Java Style Guide: Method names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9_]+$"/>
-            <message key="name.invalidPattern" value="Method name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9_]++$"/>
         </module>
         <module name="MethodParamPad"/> <!-- Java Style Guide: Horizontal whitespace -->
         <module name="MissingDeprecated"/> <!-- Java Coding Guide: Deprecate per annotation and Javadoc -->
@@ -294,8 +304,7 @@
         <module name="PackageAnnotation"/> <!-- Java Style Guide: Package statement -->
         <module name="PackageDeclaration"/> <!-- Java Style Guide: Package statement -->
         <module name="PackageName"> <!-- Java Style Guide: Package names -->
-            <property name="format" value="^[a-z]+(\.[a-z][a-z0-9]*)*$"/>
-            <message key="name.invalidPattern" value="Package name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z]++(\.[a-z][a-z0-9]*+)*+$"/>
         </module>
         <module name="ParameterAssignment"/> <!-- Java Coding Guidelines: Final variables and parameters -->
         <module name="ParenPad"/> <!-- Java Style Guide: Horizontal whitespace -->
@@ -405,6 +414,9 @@
             <property name="format" value="@Test\(.*expected.*\)"/>
             <property name="message" value="Prefer using Assertions.assertThatThrownBy(...).isInstanceOf(...) instead."/>
         </module>
+        <module name="IllegalToken">
+            <property name="tokens" value="LITERAL_ASSERT"/>
+        </module>
         <module name="IllegalImport">
             <property name="id" value="BanExpectedExceptionUsage"/>
             <property name="illegalClasses" value="org.junit.rules.ExpectedException"/>
@@ -415,6 +427,11 @@
             <property name="illegalPkgs" value="org.hamcrest"/>
             <message key="import.illegal" value="Prefer using org.assertj.core.api.Assertions instead."/>
         </module>
+        <module name="IllegalImport">
+            <property name="id" value="BanJUnit5Assertions"/>
+            <property name="illegalPkgs" value="org.junit.jupiter.api.Assertions"/>
+            <message key="import.illegal" value="Prefer using org.assertj.core.api.Assertions instead."/>
+        </module>
         <module name="RegexpSinglelineJava">
             <property name="ignoreComments" value="true"/>
             <property name="format" value="@Json(S|Des)erialize"/>
@@ -434,7 +451,6 @@
             <message key="todo.match" value="There must be whitespace at the beginning of all comments."/>
         </module>
         <module name="TypeName"> <!-- Java Style Guide: Class names -->
-            <message key="name.invalidPattern" value="Type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="TypecastParenPad"/> <!-- Java Style Guide: Horizontal whitespace -->
         <module name="UnusedImports"> <!-- Java Style Guide: No unused imports -->
@@ -470,23 +486,20 @@
         <module name="LocalFinalVariableName"/> <!-- Java Style Guide: Local variable names -->
         <module name="LocalVariableName"> <!-- Java Style Guide: Local variable names -->
             <property name="tokens" value="VARIABLE_DEF"/>
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
             <property name="allowOneCharVarInForLoop" value="true"/>
-            <message key="name.invalidPattern" value="Local variable name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="MethodLength"/> <!-- Java Coding Guide: Methods and functions: focused, crisp, concise -->
         <module name="MethodTypeParameterName"> <!-- Java Style Guide: Type variable names -->
             <property name="format" value="(^[A-Z][0-9]?)$|([A-Z][a-zA-Z0-9]*[T]$)"/>
-            <message key="name.invalidPattern" value="Method type name ''{0}'' must match pattern ''{1}''."/>
         </module>
         <module name="NestedForDepth">
             <property name="max" value="2"/>
         </module>
         <module name="NestedTryDepth"/> <!-- Java Coding Guide: Try/catch blocks: never nested -->
         <module name="NonEmptyAtclauseDescription"/> <!-- Java Style Guide: At-clauses -->
         <module name="ParameterName"> <!-- Java Style Guide: Parameter names -->
-            <property name="format" value="^[a-z][a-zA-Z0-9]+$"/>
-            <message key="name.invalidPattern" value="Parameter name ''{0}'' must match pattern ''{1}''."/>
+            <property name="format" value="^[a-z][a-zA-Z0-9]++$"/>
             <property name="ignoreOverridden" value="true"/>
         </module>
 

diff --git a/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml b/.github/ISSUE_TEMPLATE/iceberg_bug_report.yml
@@ -28,7 +28,12 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.4.3 (latest release)"
+        - "1.6.1 (latest release)"
+        - "1.6.0"
+        - "1.5.2"
+        - "1.5.1"
+        - "1.5.0"
+        - "1.4.3"
         - "1.4.2"
         - "1.4.1"
         - "1.4.0"
@@ -38,14 +43,6 @@ body:
         - "1.2.0"
         - "1.1.0"
         - "1.0.0"
-        - "0.14.1"
-        - "0.14.0"
-        - "0.13.1"
-        - "0.13.0"
-        - "0.12.1"
-        - "0.12.0"
-        - "0.11.1"
-        - "0.11.0"
         - "main (development)"
     validations:
       required: false
@@ -81,3 +78,11 @@ body:
         You can include files by dragging and dropping them here.
     validations:
       required: true
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages bug fix contributions. Would you or another member of your organization be willing to contribute a fix for this bug to the Apache Iceberg codebase?
+      options:
+        - label: I can contribute a fix for this bug independently
+        - label: I would be willing to contribute a fix for this bug with guidance from the Iceberg community
+        - label: I cannot contribute a fix for this bug at this time
diff --git a/.github/ISSUE_TEMPLATE/iceberg_improvement.yml b/.github/ISSUE_TEMPLATE/iceberg_improvement.yml
@@ -50,4 +50,12 @@ body:
         - Hive
         - Other
     validations:
-      required: false
+      required: false
+  - type: checkboxes
+    attributes:
+      label: Willingness to contribute
+      description: The Apache Iceberg community encourages contributions. Would you or another member of your organization be willing to contribute this improvement/feature to the Apache Iceberg codebase?
+      options:
+        - label: I can contribute this improvement/feature independently
+        - label: I would be willing to contribute this improvement/feature with guidance from the Iceberg community
+        - label: I cannot contribute this improvement/feature at this time
diff --git a/.github/ISSUE_TEMPLATE/iceberg_proposal.yml b/.github/ISSUE_TEMPLATE/iceberg_proposal.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+name: Iceberg Improvement Proposal
+description: Propose a Spec change or major feature
+labels: ["proposal"]
+body:
+  - type: markdown
+    attributes:
+      value: "Please see documentation site for information on [contributing proposals](https://iceberg.apache.org/contribute/#apache-iceberg-improvement-proposals)"
+  - type: textarea
+    attributes:
+      label: Proposed Change
+      description: Please describe the proposal and elaborate on the use case and motivation
+    validations:
+      required: true
+  - type: input
+    attributes:
+      label: Proposal document
+      description: |
+        Link to the proposal document. Google Docs is preferred format to allow for public 
+        comment and sharing
+  - type: checkboxes
+    attributes:
+      label: Specifications
+      description: Which specifications are affected by this proposal?
+      options:
+        - label: Table
+        - label: View
+        - label: REST
+        - label: Puffin
+        - label: Encryption
+        - label: Other
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -24,16 +24,24 @@ updates:
     schedule:
       interval: "weekly"
       day: "sunday"
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
   - package-ecosystem: "gradle"
     directory: "/"
     schedule:
       interval: "weekly"
       day: "sunday"
     open-pull-requests-limit: 50
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
       interval: "weekly"
       day: "sunday"
     open-pull-requests-limit: 5
-
+    ignore:
+      - dependency-name: "*"
+        update-types: ["version-update:semver-major"]
diff --git a/.github/workflows/api-binary-compatibility.yml b/.github/workflows/api-binary-compatibility.yml
@@ -22,11 +22,16 @@ on:
   push:
     branches:
       - 'main'
-      - '0.**'
+      - '0.*'
+      - '1.*'
+      - '2.*'
     tags:
       - 'apache-iceberg-**'
   pull_request:
     paths:
+      - '.github/workflows/api-binary-compatibility.yml'
+      - '*.gradle'
+      - 'gradle*'
       - 'api/**'
       - '.palantir/revapi.yml'
 

diff --git a/.github/workflows/delta-conversion-ci.yml b/.github/workflows/delta-conversion-ci.yml
@@ -22,7 +22,9 @@ on:
   push:
     branches:
       - 'main'
-      - '0.**'
+      - '0.*'
+      - '1.*'
+      - '2.*'
     tags:
       - 'apache-iceberg-**'
   pull_request:
@@ -33,6 +35,7 @@ on:
       - '.github/workflows/hive-ci.yml'
       - '.github/workflows/java-ci.yml'
       - '.github/workflows/jmh-benchmarks-ci.yml'
+      - '.github/workflows/kafka-connect-ci.yml'
       - '.github/workflows/labeler.yml'
       - '.github/workflows/licence-check.yml'
       - '.github/workflows/open-api.yml'
@@ -49,6 +52,7 @@ on:
       - 'hive3-orc-bundle/**'
       - 'hive-runtime/**'
       - 'flink/**'
+      - 'kafka-connect/**'
       - 'pig/**'
       - 'docs/**'
       - 'site/**'
@@ -69,7 +73,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -86,7 +90,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:
@@ -98,7 +102,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11, 17]
+        jvm: [11, 17, 21]
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -115,7 +119,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

diff --git a/.github/workflows/flink-ci.yml b/.github/workflows/flink-ci.yml
@@ -22,7 +22,9 @@ on:
   push:
     branches:
     - 'main'
-    - '0.**'
+    - '0.*'
+    - '1.*'
+    - '2.*'
     tags:
     - 'apache-iceberg-**'
   pull_request:
@@ -33,6 +35,7 @@ on:
     - '.github/workflows/hive-ci.yml'
     - '.github/workflows/java-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
     - '.github/workflows/licence-check.yml'
     - '.github/workflows/open-api.yml'
@@ -48,6 +51,7 @@ on:
     - 'hive3/**'
     - 'hive3-orc-bundle/**'
     - 'hive-runtime/**'
+    - 'kafka-connect/**'
     - 'spark/**'
     - 'pig/**'
     - 'docs/**'
@@ -71,8 +75,8 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        jvm: [8, 11]
-        flink: ['1.16', '1.17', '1.18']
+        jvm: [11, 17, 21]
+        flink: ['1.18', '1.19', '1.20']
     env:
       SPARK_LOCAL_IP: localhost
     steps:
@@ -89,7 +93,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions= -DkafkaVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with: