Merge remote-tracking branch 'origin/main' into lint-dcheck-macros

.asf.yaml

@@ -21,9 +21,9 @@ github:
   collaborators:
     - anjakefala
     - benibus
-    - davisusanibar
     - jbonofre
     - js8544
+    - laurentgo
     - vibhatha
     - zanmato1984
     - ZhangHuiGui

.github/CONTRIBUTING.md

@@ -40,7 +40,7 @@ We prefer to receive contributions in the form of GitHub pull requests. Please
 send pull requests against the [github.com/apache/arrow][4] repository following
 the procedure below.
 
-If you are looking for some ideas on what to contribute, check out the [JIRA
+If you are looking for some ideas on what to contribute, check out the [GitHub
 issues][3] for the Apache Arrow project. Comment on the issue and/or contact
 [dev@arrow.apache.org](https://lists.apache.org/list.html?dev@arrow.apache.org)
 with your questions and ideas.
@@ -53,8 +53,8 @@ To contribute a patch:
 
 1. Break your work into small, single-purpose patches if possible. It’s much
 harder to merge in a large change with a lot of disjoint features.
-2. If one doesn't already exist, create a JIRA for your patch on the [Arrow Project
-JIRA](https://issues.apache.org/jira/browse/ARROW).
+2. If one doesn't already exist, create a GitHub issue for your patch on the [Arrow Project
+GitHub](https://github.com/apache/arrow/issues).
 3. Submit the patch as a GitHub pull request against the main branch. For a
 tutorial, see the GitHub guides on [forking a repo](https://help.github.com/en/articles/fork-a-repo)
 and [sending a pull request](https://help.github.com/en/articles/creating-a-pull-request-from-a-fork). So that your pull request syncs with the JIRA issue, prefix your pull request
@@ -68,5 +68,5 @@ Thank you in advance for your contributions!
 
 [1]: mailto:dev-subscribe@arrow.apache.org
 [2]: https://github.com/apache/arrow/tree/main/format
-[3]: https://issues.apache.org/jira/browse/ARROW
+[3]: https://github.com/apache/arrow/issues
 [4]: https://github.com/apache/arrow

.github/workflows/go.yml

@@ -168,8 +168,8 @@ jobs:
           python3 -m pip install benchadapt@git+https://github.com/conbench/conbench.git@main#subdirectory=benchadapt/python
           python3 ci/scripts/go_bench_adapt.py
 
-  build386:
-    name: Go Cross-build for 386
+  build_test_386:
+    name: Go Cross-build and test for 386
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 20
@@ -188,9 +188,12 @@ jobs:
           cache: true
           cache-dependency-path: go/go.sum
       - name: Run build
-        run: |
-          cd go
-          GOARCH=386 go build ./...
+        run: GOARCH=386 go build ./...
+        working-directory: ./go
+      - name: Run test
+        # WIP refactor, only tests in the specified dirs have been fixed
+        run: GOARCH=386 go test ./parquet/file/...
+        working-directory: ./go
 
   docker_cgo:
     name: AMD64 Debian 12 Go ${{ matrix.go }} - CGO

.github/workflows/r.yml

@@ -62,7 +62,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - cpp_version: "13.0.0"
+          - cpp_version: "15.0.2"
     steps:
     - name: Checkout Arrow
       uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0

.pre-commit-config.yaml

@@ -38,6 +38,7 @@ repos:
         # files: (/Dockerfile|\.dockerfile)$
         files: >-
           (
+          ?^ci/docker/conda-python-emscripten\.dockerfile$|
           ?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
           )
         types: []

ci/docker/centos-7-cpp.dockerfile

@@ -17,11 +17,25 @@
 
 FROM centos:centos7
 
+# Update mirrors to use vault.centos.org as CentOS 7
+# is EOL since 2024-06-30
+RUN sed -i \
+      -e 's/^mirrorlist/#mirrorlist/' \
+      -e 's/^#baseurl/baseurl/' \
+      -e 's/mirror\.centos\.org/vault.centos.org/' \
+      /etc/yum.repos.d/*.repo
+
 # devtoolset is required for C++17
 RUN \
   yum install -y \
     centos-release-scl \
     epel-release && \
+  sed -i \
+    -e 's/^mirrorlist/#mirrorlist/' \
+    -e 's/^#baseurl/baseurl/' \
+    -e 's/^# baseurl/baseurl/' \
+    -e 's/mirror\.centos\.org/vault.centos.org/' \
+    /etc/yum.repos.d/CentOS-SCLo-scl*.repo && \
   yum install -y \
     cmake3 \
     curl \

ci/docker/conda-python-emscripten.dockerfile

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG repo
+ARG arch
+ARG python="3.12"
+FROM ${repo}:${arch}-conda-python-${python}
+
+ARG selenium_version="4.15.2"
+ARG pyodide_version="0.26.0"
+ARG chrome_version="latest"
+ARG required_python_min="(3,12)"
+# fail if python version < 3.12
+RUN echo "check PYTHON>=${required_python_min}" && python -c "import sys;sys.exit(0 if sys.version_info>=${required_python_min} else 1)"
+
+# install selenium and pyodide-build and recent python
+
+# needs to be a login shell so ~/.profile is read
+SHELL ["/bin/bash", "--login", "-c", "-o", "pipefail"]
+
+RUN python -m pip install --no-cache-dir selenium==${selenium_version} && \
+    python -m pip install --no-cache-dir --upgrade pyodide-build==${pyodide_version}
+
+# install pyodide dist directory to /pyodide
+RUN pyodide_dist_url="https://github.com/pyodide/pyodide/releases/download/${pyodide_version}/pyodide-${pyodide_version}.tar.bz2" && \
+    wget -q "${pyodide_dist_url}" -O- | tar -xj -C /
+
+# install correct version of emscripten for this pyodide
+COPY ci/scripts/install_emscripten.sh /arrow/ci/scripts/
+RUN bash /arrow/ci/scripts/install_emscripten.sh ~ /pyodide
+
+# make sure zlib is cached in the EMSDK folder
+RUN source ~/emsdk/emsdk_env.sh && embuilder --pic build zlib
+
+# install node 20 (needed for async call support)
+# and pthread-stubs for build, and unzip needed for chrome build to work
+RUN conda install nodejs=20  unzip pthread-stubs make -c conda-forge
+
+# install chrome for testing browser based runner
+COPY ci/scripts/install_chromedriver.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_chromedriver.sh "${chrome_version}"
+
+# make the version of make that is installed by conda be available everywhere
+# or else pyodide's isolated build fails to find it
+RUN ln -s "$(type -P make)" /bin/make
+
+ENV ARROW_BUILD_TESTS="OFF" \
+    ARROW_BUILD_TYPE="release" \
+    ARROW_DEPENDENCY_SOURCE="BUNDLED" \
+    ARROW_EMSCRIPTEN="ON"

ci/docker/fedora-39-cpp.dockerfile

@@ -77,8 +77,7 @@ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
 # PYARROW_TEST_GANDIVA=OFF: GH-39695: We need to make LLVM symbols visible in
 # Python process explicitly if we use LLVM 17 or later.
-ENV absl_SOURCE=BUNDLED \
-    ARROW_ACERO=ON \
+ENV ARROW_ACERO=ON \
     ARROW_AZURE=OFF \
     ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/docker/python-wheel-manylinux.dockerfile

@@ -25,6 +25,18 @@ ARG manylinux
 ENV MANYLINUX_VERSION=${manylinux}
 
 # Ensure dnf is installed, especially for the manylinux2014 base
+RUN if [ "${MANYLINUX_VERSION}" = "2014" ]; then \
+      sed -i \
+        -e 's/^mirrorlist/#mirrorlist/' \
+        -e 's/^#baseurl/baseurl/' \
+        -e 's/mirror\.centos\.org/vault.centos.org/' \
+        /etc/yum.repos.d/*.repo; \
+      if [ "${arch}" != "amd64" ]; then \
+        sed -i \
+          -e 's,vault\.centos\.org/centos,vault.centos.org/altarch,' \
+          /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo; \
+      fi; \
+    fi
 RUN yum install -y dnf
 
 # Install basic dependencies

ci/scripts/PKGBUILD

@@ -69,6 +69,12 @@ build() {
   mkdir -p ${cpp_build_dir}
   pushd ${cpp_build_dir}
 
+  # We use static cURL in google-cloud-cpp. If we can use cURL's CMake
+  # package, we don't need to specify CURL_STATICLIB explicitly. But
+  # we don't have cURL's CMake package. We need to use CXXFLAGS
+  # instead of ARROW_CXXFLAGS because ARROW_CXXFLAGS aren't passed to
+  # ExternProjects.
+  export CXXFLAGS="${CXXFLAGS} -DCURL_STATICLIB"
   # The Rtools libutf8proc is a static lib, but Findutf8proc.cmake doesn't
   # set the appropriate compiler definition.
   export CPPFLAGS="-DUTF8PROC_STATIC"

ci/scripts/cpp_build.sh

@@ -30,7 +30,7 @@ if [ -x "$(command -v git)" ]; then
 fi
 
 # TODO(kszucs): consider to move these to CMake
-if [ ! -z "${CONDA_PREFIX}" ]; then
+if [ ! -z "${CONDA_PREFIX}" ] && [ "${ARROW_EMSCRIPTEN:-OFF}" = "OFF" ]; then
   echo -e "===\n=== Conda environment for build\n==="
   conda list
 
@@ -99,6 +99,10 @@ if [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then
   fi
   n_jobs=2 # Emscripten build fails on docker unless this is set really low
   source ~/emsdk/emsdk_env.sh
+  export CMAKE_INSTALL_PREFIX=$(em-config CACHE)/sysroot
+  # conda sets LDFLAGS / CFLAGS etc. which break
+  # emcmake so we unset them
+  unset LDFLAGS CFLAGS CXXFLAGS CPPFLAGS
   emcmake cmake \
     --preset=ninja-${ARROW_BUILD_TYPE:-debug}-emscripten \
     -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \

ci/scripts/install_chromedriver.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Install Chrome and Chromedriver for Selenium
+
+set -e
+
+chrome_version=$1
+
+if [ $chrome_version = "latest" ]; then
+  latest_release_path=LATEST_RELEASE_STABLE
+else
+  latest_release_path=LATEST_RELEASE_${chrome_version}
+fi
+CHROME_VERSION_FULL=$(wget -q --no-verbose -O - "https://googlechromelabs.github.io/chrome-for-testing/${latest_release_path}")
+CHROME_DOWNLOAD_URL="https://dl.google.com/linux/chrome/deb/pool/main/g/google-chrome-stable/google-chrome-stable_${CHROME_VERSION_FULL}-1_amd64.deb"
+CHROMEDRIVER_DOWNLOAD_URL="https://storage.googleapis.com/chrome-for-testing-public/${CHROME_VERSION_FULL}/linux64/chromedriver-linux64.zip"
+wget -q --no-verbose -O /tmp/google-chrome.deb "${CHROME_DOWNLOAD_URL}"
+apt-get update
+apt install -qqy /tmp/google-chrome.deb
+rm -f /tmp/google-chrome.deb
+rm -rf /var/lib/apt/lists/*
+wget --no-verbose -O /tmp/chromedriver-linux64.zip "${CHROMEDRIVER_DOWNLOAD_URL}"
+unzip /tmp/chromedriver-linux64.zip -d /opt/
+rm /tmp/chromedriver-linux64.zip
+ln -fs /opt/chromedriver-linux64/chromedriver /usr/local/bin/chromedriver
+echo "Using Chrome version: $(google-chrome --version)"
+echo "Using Chrome Driver version: $(chromedriver --version)"

ci/scripts/install_emscripten.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# install emscripten sdk version to match pyodide in $2 to directory $1/emsdk
+
+set -e
+
+target_path=$1
+pyodide_path=$2
+
+emscripten_version=$(${pyodide_path}/python -c "import sys;print(*sys._emscripten_info.emscripten_version,sep='.')")
+
+cd ${target_path}
+if [ ! -d emsdk ]; then
+  git clone https://github.com/emscripten-core/emsdk.git
+fi
+cd emsdk 
+./emsdk install ${emscripten_version} 
+./emsdk activate ${emscripten_version}
+echo "Installed emsdk to: ${target_path}"

ci/scripts/python_build_emscripten.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+arrow_dir=${1}
+build_dir=${2}
+
+
+source ~/emsdk/emsdk_env.sh
+
+source_dir=${arrow_dir}/python
+python_build_dir=${build_dir}/python
+
+rm -rf ${python_build_dir}
+cp -aL ${source_dir} ${python_build_dir}
+
+# conda sets LDFLAGS / CFLAGS etc. which break
+# emcmake so we unset them
+unset LDFLAGS CFLAGS CXXFLAGS CPPFLAGS
+
+pushd ${python_build_dir}
+pyodide build
+popd

ci/scripts/python_test_emscripten.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# run tests against Chrome and node.js as representative
+# WebAssembly platforms (i.e. one browser, one non-browser).
+
+set -ex
+
+build_dir=${1}/python
+pyodide_dist_dir=${2}
+
+cd ${build_dir}
+
+# note: this uses the newest wheel in dist
+pyodide_wheel=$(ls -t dist/pyarrow*.whl | head -1)
+
+echo "-------------- Running emscripten tests in Node ----------------------"
+python scripts/run_emscripten_tests.py ${pyodide_wheel} --dist-dir=${pyodide_dist_dir} --runtime=node
+
+echo "-------------- Running emscripten tests in Chrome --------------------"
+python scripts/run_emscripten_tests.py ${pyodide_wheel} --dist-dir=${pyodide_dist_dir} --runtime=chrome
+