Merge upstream

.asf.yaml

@@ -25,7 +25,6 @@ github:
     - js8544
     - laurentgo
     - vibhatha
-    - zanmato1984
     - ZhangHuiGui
 
 notifications:

.env

@@ -97,7 +97,7 @@ VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01"    # 2024.04.26 Release
 # use pulled built images in dev/tasks/python-wheels/github.windows.yml.
 PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-08-06
 
-# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker-compose run --rm conan".
+# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm conan".
 # See https://github.com/conan-io/conan-docker-tools#readme and
 # https://hub.docker.com/u/conanio for available images.
 CONAN_BASE=gcc10

.github/CODEOWNERS

@@ -46,7 +46,6 @@
 
 # Docs
 # /docs/
-# .readthedocs.yml
 # *.md
 # *.rmd
 # *.rst

.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -38,15 +38,15 @@ body:
         - C
         - C#
         - C++
-        - C++ - Gandiva
         - Continuous Integration
         - Developer Tools
         - Documentation
         - FlightRPC
         - Format
         - GLib
-        - Go
         - GPU
+        - Gandiva
+        - Go
         - Integration
         - Java
         - JavaScript

.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -41,15 +41,15 @@ body:
         - C
         - C#
         - C++
-        - C++ - Gandiva
         - Continuous Integration
         - Developer Tools
         - Documentation
         - FlightRPC
         - Format
         - GLib
-        - Go
         - GPU
+        - Gandiva
+        - Go
         - Integration
         - Java
         - JavaScript

.github/ISSUE_TEMPLATE/usage_question.yaml

@@ -61,15 +61,15 @@ body:
         - C
         - C#
         - C++
-        - C++ - Gandiva
         - Continuous Integration
         - Developer Tools
         - Documentation
         - FlightRPC
         - Format
         - GLib
-        - Go
         - GPU
+        - Gandiva
+        - Go
         - Integration
         - Java
         - JavaScript

.github/workflows/dev_pr/labeler.yml

@@ -53,11 +53,12 @@
   - r/R/flight.*
   - python/pyarrow/*flight.*
 
-"Component: C++ - Gandiva":
+"Component: Gandiva":
   - c_glib/gandiva-glib/**/*
   - cpp/src/gandiva/**/*
-  - ruby/red-gandiva/**/*
+  - java/gandiva/**/*
   - python/pyarrow/gandiva.*
+  - ruby/red-gandiva/**/*
 
 "Component: Parquet":
   - c_glib/parquet-glib/**/*

.github/workflows/matlab.yml

@@ -75,7 +75,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2024a
+          release: R2024b
       - name: Install ccache
         run: sudo apt-get install ccache
       - name: Setup ccache
@@ -123,7 +123,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2024a
+          release: R2024b
       - name: Install ccache
         run: brew install ccache
       - name: Setup ccache
@@ -162,7 +162,7 @@ jobs:
       - name: Install MATLAB
         uses: matlab-actions/setup-matlab@v2
         with:
-          release: R2024a
+          release: R2024b
       - name: Download Timezone Database
         shell: bash
         run: ci/scripts/download_tz_database.sh

.pre-commit-config.yaml

@@ -173,3 +173,13 @@ repos:
           '--disable',
           'dangling-hyphen,line-too-long',
         ]
+  - repo: https://github.com/koalaman/shellcheck-precommit
+    rev: v0.10.0
+    hooks:
+      - id: shellcheck
+        # TODO: Remove this when we fix all lint failures
+        files: >-
+          (
+          ?^ci/scripts/c_glib_build\.sh$|
+          ?^ci/scripts/c_glib_test\.sh$|
+          )

c_glib/parquet-glib/arrow-file-reader.cpp

@@ -134,12 +134,13 @@ gparquet_arrow_file_reader_new_arrow(GArrowSeekableInputStream *source, GError *
 {
   auto arrow_random_access_file = garrow_seekable_input_stream_get_raw(source);
   auto arrow_memory_pool = arrow::default_memory_pool();
-  std::unique_ptr<parquet::arrow::FileReader> parquet_arrow_file_reader;
-  auto status = parquet::arrow::OpenFile(arrow_random_access_file,
-                                         arrow_memory_pool,
-                                         &parquet_arrow_file_reader);
-  if (garrow_error_check(error, status, "[parquet][arrow][file-reader][new-arrow]")) {
-    return gparquet_arrow_file_reader_new_raw(parquet_arrow_file_reader.release());
+  auto parquet_arrow_file_reader_result =
+    parquet::arrow::OpenFile(arrow_random_access_file, arrow_memory_pool);
+  if (garrow::check(error,
+                    parquet_arrow_file_reader_result,
+                    "[parquet][arrow][file-reader][new-arrow]")) {
+    return gparquet_arrow_file_reader_new_raw(
+      parquet_arrow_file_reader_result->release());
   } else {
     return NULL;
   }
@@ -168,12 +169,13 @@ gparquet_arrow_file_reader_new_path(const gchar *path, GError **error)
   std::shared_ptr<arrow::io::RandomAccessFile> arrow_random_access_file =
     arrow_memory_mapped_file.ValueOrDie();
   auto arrow_memory_pool = arrow::default_memory_pool();
-  std::unique_ptr<parquet::arrow::FileReader> parquet_arrow_file_reader;
-  auto status = parquet::arrow::OpenFile(arrow_random_access_file,
-                                         arrow_memory_pool,
-                                         &parquet_arrow_file_reader);
-  if (garrow::check(error, status, "[parquet][arrow][file-reader][new-path]")) {
-    return gparquet_arrow_file_reader_new_raw(parquet_arrow_file_reader.release());
+  auto parquet_arrow_file_reader_result =
+    parquet::arrow::OpenFile(arrow_random_access_file, arrow_memory_pool);
+  if (garrow::check(error,
+                    parquet_arrow_file_reader_result,
+                    "[parquet][arrow][file-reader][new-path]")) {
+    return gparquet_arrow_file_reader_new_raw(
+      parquet_arrow_file_reader_result->release());
   } else {
     return NULL;
   }

ci/docker/python-wheel-manylinux.dockerfile

@@ -69,36 +69,47 @@ RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 ARG vcpkg
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
+     ci/vcpkg/vcpkg.json \
      arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh \
      arrow/ci/scripts/
 ENV VCPKG_ROOT=/opt/vcpkg
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
-    VCPKG_FORCE_SYSTEM_BINARIES=1 \
-    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
+    PATH="${PATH}:${VCPKG_ROOT}" \
     VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
-    VCPKG_FEATURE_FLAGS="manifests"
-
-RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
-ENV PATH="${PATH}:${VCPKG_ROOT}"
-
-COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
-# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
-# ssl related fixes as well as we can patch the vcpkg portfile to support
-# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
-# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
-# previous versions => use bundled S3 build
-RUN vcpkg install \
+    VCPKG_FEATURE_FLAGS="manifests" \
+    VCPKG_FORCE_SYSTEM_BINARIES=1 \
+    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg
+# For --mount=type=secret: The GITHUB_TOKEN is the only real secret but we use
+# --mount=type=secret for GITHUB_REPOSITORY_OWNER and
+# VCPKG_BINARY_SOURCES too because we don't want to store them
+# into the built image in order to easily reuse the built image cache.
+#
+# For vcpkg install: cannot use the S3 feature here because while
+# aws-sdk-cpp=1.9.160 contains ssl related fixes as well as we can
+# patch the vcpkg portfile to support arm machines it hits ARROW-15141
+# where we would need to fall back to 1.8.186 but we cannot patch
+# those portfiles since vcpkg-tool handles the checkout of previous
+# versions => use bundled S3 build
+RUN --mount=type=secret,id=github_repository_owner \
+    --mount=type=secret,id=github_token \
+    --mount=type=secret,id=vcpkg_binary_sources \
+      export GITHUB_REPOSITORY_OWNER=$(cat /run/secrets/github_repository_owner); \
+      export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
+      export VCPKG_BINARY_SOURCES=$(cat /run/secrets/vcpkg_binary_sources); \
+      arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
+      vcpkg install \
         --clean-after-build \
         --x-install-root=${VCPKG_ROOT}/installed \
         --x-manifest-root=/arrow/ci/vcpkg \
-        --x-feature=azure \ 
+        --x-feature=azure \
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \
         --x-feature=parquet \
-        --x-feature=s3
+        --x-feature=s3 && \
+      rm -rf ~/.config/NuGet/
 
 # Make sure auditwheel is up-to-date
 RUN pipx upgrade auditwheel

ci/scripts/c_glib_build.sh

@@ -23,59 +23,63 @@ source_dir=${1}/c_glib
 build_dir=${2}/c_glib
 build_root=${2}
 
-: ${ARROW_GLIB_WERROR:=false}
-: ${ARROW_GLIB_VAPI:=true}
-: ${BUILD_DOCS_C_GLIB:=OFF}
+: "${ARROW_GLIB_WERROR:=false}"
+: "${ARROW_GLIB_VAPI:=true}"
+: "${BUILD_DOCS_C_GLIB:=OFF}"
 with_doc=$([ "${BUILD_DOCS_C_GLIB}" == "ON" ] && echo "true" || echo "false")
 
 
 if [ -n "${MSYSTEM:-}" ]; then
   # Fix ARROW_HOME when running under MSYS2
-  export ARROW_HOME="$(cygpath --unix "${ARROW_HOME}")"
+  ARROW_HOME="$(cygpath --unix "${ARROW_HOME}")"
+  export ARROW_HOME
 fi
 
-export PATH="${ARROW_HOME}/bin:${PATH}"
+PATH="${ARROW_HOME}/bin:${PATH}"
 
 meson_pkg_config_path="${ARROW_HOME}/lib/pkgconfig"
 
-mkdir -p ${build_dir}
+mkdir -p "${build_dir}"
 
-if [ -n "${VCPKG_ROOT:-}" -a -n "${VCPKG_TRIPLET:-}" ]; then
+if [ -n "${VCPKG_ROOT:-}" ] && [ -n "${VCPKG_TRIPLET:-}" ]; then
   vcpkg_install_root="${build_root}/vcpkg_installed"
-  $VCPKG_ROOT/vcpkg install --x-manifest-root=${source_dir} --x-install-root=${vcpkg_install_root}
-  export PKG_CONFIG="${vcpkg_install_root}/${VCPKG_TRIPLET}/tools/pkgconf/pkgconf.exe"
+  "${VCPKG_ROOT}/vcpkg" install \
+    --x-manifest-root="${source_dir}" \
+    --x-install-root="${vcpkg_install_root}"
+  PKG_CONFIG="${vcpkg_install_root}/${VCPKG_TRIPLET}/tools/pkgconf/pkgconf.exe"
+  export PKG_CONFIG
   meson_pkg_config_path="${vcpkg_install_root}/${VCPKG_TRIPLET}/lib/pkgconfig:${meson_pkg_config_path}"
   # Configure PATH for libraries required by the gobject-introspection generated binary
   cpp_vcpkg_install_root="${build_root}/cpp/vcpkg_installed"
   PATH="${cpp_vcpkg_install_root}/${VCPKG_TRIPLET}/debug/bin:${PATH}"
   PATH="${cpp_vcpkg_install_root}/${VCPKG_TRIPLET}/bin:${PATH}"
-  export PATH="${vcpkg_install_root}/${VCPKG_TRIPLET}/bin:${PATH}"
+  PATH="${vcpkg_install_root}/${VCPKG_TRIPLET}/bin:${PATH}"
 fi
 
-if [ -n "${VCToolsInstallDir:-}" -a -n "${MSYSTEM:-}" ]; then
+if [ -n "${VCToolsInstallDir:-}" ] && [ -n "${MSYSTEM:-}" ]; then
   # Meson finds the gnu link.exe instead of MSVC link.exe when running in MSYS2/git bash,
   # so we need to make sure the MSCV link.exe is first in $PATH
-  export PATH="$(cygpath --unix "${VCToolsInstallDir}")/bin/HostX64/x64:${PATH}"
+  PATH="$(cygpath --unix "${VCToolsInstallDir}")/bin/HostX64/x64:${PATH}"
 fi
 
 # Build with Meson
 meson setup \
       --backend=ninja \
-      --prefix=$ARROW_HOME \
+      --prefix="${ARROW_HOME}" \
       --libdir=lib \
       --pkg-config-path="${meson_pkg_config_path}" \
-      -Ddoc=${with_doc} \
-      -Dvapi=${ARROW_GLIB_VAPI} \
-      -Dwerror=${ARROW_GLIB_WERROR} \
-      ${build_dir} \
-      ${source_dir}
+      -Ddoc="${with_doc}" \
+      -Dvapi="${ARROW_GLIB_VAPI}" \
+      -Dwerror="${ARROW_GLIB_WERROR}" \
+      "${build_dir}" \
+      "${source_dir}"
 
-pushd ${build_dir}
+pushd "${build_dir}"
 ninja
 ninja install
 popd
 
 if [ "${BUILD_DOCS_C_GLIB}" == "ON" ]; then
-  mkdir -p ${build_root}/docs/c_glib
-  cp -a ${ARROW_HOME}/share/doc/*-glib/ ${build_root}/docs/c_glib/
+  mkdir -p "${build_root}/docs/c_glib"
+  cp -a "${ARROW_HOME}"/share/doc/*-glib/ "${build_root}/docs/c_glib/"
 fi

ci/scripts/c_glib_test.sh

@@ -22,7 +22,7 @@ set -ex
 source_dir=${1}/c_glib
 build_dir=${2}/c_glib
 
-: ${ARROW_GLIB_VAPI:=true}
+: "${ARROW_GLIB_VAPI:=true}"
 
 export DYLD_LIBRARY_PATH=${ARROW_HOME}/lib:${DYLD_LIBRARY_PATH}
 export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
@@ -34,7 +34,7 @@ if [ -z "${ARROW_DEBUG_MEMORY_POOL}" ]; then
   export ARROW_DEBUG_MEMORY_POOL=trap
 fi
 
-pushd ${source_dir}
+pushd "${source_dir}"
 
 ruby test/run-test.rb
 
@@ -51,7 +51,7 @@ fi
 
 popd
 
-pushd ${build_dir}
+pushd "${build_dir}"
 example/build
 example/extension-type
 if [ "${ARROW_GLIB_VAPI}" = "true" ]; then