Merge remote-tracking branch 'upstream/main'

dariocurr · dariocurr · commit 4fb3b02cffe3 · 2026-02-26T10:34:44.000+01:00
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -29,6 +29,6 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.14"
       - name: Audit licenses
         run: ./dev/release/run-rat.sh .
diff --git a/.github/workflows/verify-release-candidate.yml b/.github/workflows/verify-release-candidate.yml
@@ -0,0 +1,78 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Verify Release Candidate
+
+# NOTE: This workflow is intended to be run manually via workflow_dispatch.
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: Version number (e.g., 52.0.0)
+        required: true
+        type: string
+      rc_number:
+        description: Release candidate number (e.g., 0)
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  verify:
+    name: Verify RC (${{ matrix.os }}-${{ matrix.arch }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Linux
+          - os: linux
+            arch: x64
+            runner: ubuntu-latest
+          - os: linux
+            arch: arm64
+            runner: ubuntu-24.04-arm
+
+          # macOS
+          - os: macos
+            arch: arm64
+            runner: macos-latest
+          - os: macos
+            arch: x64
+            runner: macos-15-intel
+
+          # Windows
+          - os: windows
+            arch: x64
+            runner: windows-latest
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up protoc
+        uses: arduino/setup-protoc@v3
+        with:
+          version: "27.4"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Run release candidate verification
+        shell: bash
+        run: ./dev/release/verify-release-candidate.sh "${{ inputs.version }}" "${{ inputs.rc_number }}"
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -17,7 +17,7 @@
 
 [package]
 name = "datafusion-python"
-version = "51.0.0"
+version = "52.0.0"
 homepage = "https://datafusion.apache.org/python"
 repository = "https://github.com/apache/datafusion-python"
 authors = ["Apache DataFusion <dev@datafusion.apache.org>"]
diff --git a/dev/changelog/52.0.0.md b/dev/changelog/52.0.0.md
@@ -0,0 +1,78 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Apache DataFusion Python 52.0.0 Changelog
+
+This release consists of 26 commits from 9 contributors. See credits at the end of this changelog for more information.
+
+**Implemented enhancements:**
+
+- feat: add CatalogProviderList support [#1363](https://github.com/apache/datafusion-python/pull/1363) (timsaucer)
+- feat: add support for generating JSON formatted substrait plan [#1376](https://github.com/apache/datafusion-python/pull/1376) (Prathamesh9284)
+- feat: add regexp_instr function [#1382](https://github.com/apache/datafusion-python/pull/1382) (mesejo)
+
+**Fixed bugs:**
+
+- fix: mangled errors [#1377](https://github.com/apache/datafusion-python/pull/1377) (mesejo)
+
+**Documentation updates:**
+
+- docs: Clarify first_value usage in select vs aggregate [#1348](https://github.com/apache/datafusion-python/pull/1348) (AdMub)
+
+**Other:**
+
+- Release 51.0.0 [#1333](https://github.com/apache/datafusion-python/pull/1333) (timsaucer)
+- Use explicit timer in unit test [#1338](https://github.com/apache/datafusion-python/pull/1338) (timsaucer)
+- Add use_fabric_endpoint parameter to MicrosoftAzure class [#1357](https://github.com/apache/datafusion-python/pull/1357) (djouallah)
+- Prepare for DF52 release [#1337](https://github.com/apache/datafusion-python/pull/1337) (timsaucer)
+- build(deps): bump actions/checkout from 5 to 6 [#1310](https://github.com/apache/datafusion-python/pull/1310) (dependabot[bot])
+- build(deps): bump actions/download-artifact from 5 to 7 [#1321](https://github.com/apache/datafusion-python/pull/1321) (dependabot[bot])
+- build(deps): bump actions/upload-artifact from 4 to 6 [#1322](https://github.com/apache/datafusion-python/pull/1322) (dependabot[bot])
+- build(deps): bump actions/cache from 4 to 5 [#1323](https://github.com/apache/datafusion-python/pull/1323) (dependabot[bot])
+- Pass Field information back and forth when using scalar UDFs [#1299](https://github.com/apache/datafusion-python/pull/1299) (timsaucer)
+- Update dependency minor versions to prepare for DF52 release [#1368](https://github.com/apache/datafusion-python/pull/1368) (timsaucer)
+- Improve displayed error by using `DataFusionError`'s `Display` trait [#1370](https://github.com/apache/datafusion-python/pull/1370) (abey79)
+- Enforce DataFrame display memory limits with `max_rows` + `min_rows` constraint (deprecate `repr_rows`) [#1367](https://github.com/apache/datafusion-python/pull/1367) (kosiew)
+- Implement all CSV reader options [#1361](https://github.com/apache/datafusion-python/pull/1361) (timsaucer)
+- chore: add confirmation before tarball is released [#1372](https://github.com/apache/datafusion-python/pull/1372) (milenkovicm)
+- Build in debug mode for PRs [#1375](https://github.com/apache/datafusion-python/pull/1375) (timsaucer)
+- minor: remove ffi test wheel from distribution artifact [#1378](https://github.com/apache/datafusion-python/pull/1378) (timsaucer)
+- chore: update rust 2024 edition [#1371](https://github.com/apache/datafusion-python/pull/1371) (timsaucer)
+- Fix Python UDAF list-of-timestamps return by enforcing list-valued scalars and caching PyArrow types [#1347](https://github.com/apache/datafusion-python/pull/1347) (kosiew)
+- minor: update cargo dependencies [#1383](https://github.com/apache/datafusion-python/pull/1383) (timsaucer)
+- chore: bump Python version for RAT checking [#1386](https://github.com/apache/datafusion-python/pull/1386) (timsaucer)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+    13	Tim Saucer
+     4	dependabot[bot]
+     2	Daniel Mesejo
+     2	kosiew
+     1	Adisa Mubarak (AdMub)
+     1	Antoine Beyeler
+     1	Dhanashri Prathamesh Iranna
+     1	Marko Milenković
+     1	Mimoune
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
+
diff --git a/dev/release/README.md b/dev/release/README.md
@@ -154,6 +154,25 @@ This will create a file named `dist/datafusion-0.7.0.tar.gz`. Upload this to tes
 python3 -m twine upload --repository testpypi dist/datafusion-0.7.0.tar.gz
 ```
 
+### Run Verify Release Candidate Workflow
+
+Before sending the vote email, run the manually triggered GitHub Actions workflow
+"Verify Release Candidate" and confirm all matrix jobs pass across the OS/architecture matrix
+(for example, Linux, macOS, and Windows runners):
+
+1. Go to https://github.com/apache/datafusion-python/actions/workflows/verify-release-candidate.yml
+2. Click "Run workflow"
+3. Set `version` to the release version (for example, `52.0.0`)
+4. Set `rc_number` to the RC number (for example, `0`)
+5. Wait for all jobs to complete successfully
+
+Include a short note in the vote email template that this workflow was run across all OS/architecture
+matrix entries and that all jobs passed.
+
+```text
+Verification note: The manually triggered "Verify Release Candidate" workflow was run for version <VERSION> and rc_number <RC_NUMBER> across all configured OS/architecture matrix entries, and all matrix jobs completed successfully.
+```
+
 ### Send the Email
 
 Send the email to start the vote.
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -112,8 +112,17 @@ test_source_distribution() {
 
   curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
 
-  export PATH=$RUSTUP_HOME/bin:$PATH
-  source $RUSTUP_HOME/env
+  # On Unix, rustup creates an env file. On Windows GitHub runners (MSYS bash),
+  # that file may not exist, so fall back to adding Cargo bin directly.
+  if [ -f "$CARGO_HOME/env" ]; then
+    # shellcheck disable=SC1090
+    source "$CARGO_HOME/env"
+  elif [ -f "$RUSTUP_HOME/env" ]; then
+    # shellcheck disable=SC1090
+    source "$RUSTUP_HOME/env"
+  else
+    export PATH="$CARGO_HOME/bin:$PATH"
+  fi
 
   # build and test rust
 
@@ -126,10 +135,20 @@ test_source_distribution() {
   git clone https://github.com/apache/parquet-testing.git parquet-testing
 
   python3 -m venv .venv
-  source .venv/bin/activate
-  python3 -m pip install -U pip
-  python3 -m pip install -U maturin
-  maturin develop
+  if [ -x ".venv/bin/python" ]; then
+    VENV_PYTHON=".venv/bin/python"
+  elif [ -x ".venv/Scripts/python.exe" ]; then
+    VENV_PYTHON=".venv/Scripts/python.exe"
+  elif [ -x ".venv/Scripts/python" ]; then
+    VENV_PYTHON=".venv/Scripts/python"
+  else
+    echo "Unable to find python executable in virtual environment"
+    exit 1
+  fi
+
+  "$VENV_PYTHON" -m pip install -U pip
+  "$VENV_PYTHON" -m pip install -U maturin
+  "$VENV_PYTHON" -m maturin develop
 
   #TODO: we should really run tests here as well
   #python3 -m pytest
diff --git a/examples/datafusion-ffi-example/src/utils.rs b/examples/datafusion-ffi-example/src/utils.rs
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 use datafusion_ffi::proto::logical_extension_codec::FFI_LogicalExtensionCodec;
 use pyo3::exceptions::PyValueError;
 use pyo3::prelude::{PyAnyMethods, PyCapsuleMethods};
diff --git a/python/tests/test_udtf.py b/python/tests/test_udtf.py
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
 import pyarrow as pa
 import pyarrow.dataset as ds
 from datafusion import Expr, SessionContext, Table, udtf
diff --git a/src/options.rs b/src/options.rs
@@ -1,3 +1,20 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
 use arrow::datatypes::{DataType, Schema};
 use arrow::pyarrow::PyArrowType;
 use datafusion::prelude::CsvReadOptions;
