GH-47632: [CI][C++] Add a CI job for JNI on Linux (#47746)

### Rationale for this change

This is for preventing to break Apache Arrow Java JNI use case on Linux.

### What changes are included in this PR?

* Add a CI job that uses build options for JNI use case
* Install more packages in manylinux image that is also used by JNI build 

### Are these changes tested?

Yes.

### Are there any user-facing changes?

No.
* GitHub Issue: #47632

Authored-by: Sutou Kouhei <kou@clear-code.com>
Signed-off-by: Sutou Kouhei <kou@clear-code.com>

Author: kou

.env

@@ -37,6 +37,10 @@ DOCKER_BUILDKIT=1
 ARCH=amd64
 ARCH_ALIAS=x86_64
 ARCH_SHORT=amd64
+# For aarch64
+# ARCH=arm64v8
+# ARCH_ALIAS=aarch64
+# ARCH_SHORT=arm64
 
 # Default repository to pull and push images from
 REPO=apache/arrow-dev

.github/workflows/cpp.yml

@@ -321,7 +321,6 @@ jobs:
       BOOST_SOURCE: BUNDLED
       CMAKE_CXX_STANDARD: "17"
       CMAKE_GENERATOR: Ninja
-      CMAKE_INSTALL_LIBDIR: bin
       CMAKE_INSTALL_PREFIX: /usr
       CMAKE_UNITY_BUILD: ON
     steps:
@@ -369,11 +368,12 @@ jobs:
           call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           bash -c "ci/scripts/cpp_build.sh $(pwd) $(pwd)/build"
       - name: Test
-        shell: bash
+        shell: cmd
         run: |
+          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           # For ORC
-          export TZDIR=/c/msys64/usr/share/zoneinfo
-          ci/scripts/cpp_test.sh $(pwd) $(pwd)/build
+          set TZDIR=C:\msys64\usr\share\zoneinfo
+          bash -c "ci/scripts/cpp_test.sh $(pwd) $(pwd)/build"
 
   windows-mingw:
     name: AMD64 Windows MinGW ${{ matrix.msystem_upper }} C++

.github/workflows/cpp_extra.yml

@@ -164,6 +164,67 @@ jobs:
         continue-on-error: true
         run: archery docker push ${{ matrix.image }}
 
+  jni-linux:
+    needs: check-labels
+    name: JNI ${{ matrix.platform.runs-on }} ${{ matrix.platform.arch }}
+    runs-on: ${{ matrix.platform.runs-on }}
+    if: >-
+      needs.check-labels.outputs.force == 'true' ||
+      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') ||
+      contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: C++')
+    timeout-minutes: 240
+    permissions:
+      # This is for using GitHub Packages for vcpkg cache
+      packages: write
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - arch: "amd64"
+            runs-on: ubuntu-latest
+          - arch: "arm64v8"
+            runs-on: ubuntu-24.04-arm
+    env:
+      ARCH: ${{ matrix.platform.arch }}
+      REPO: ghcr.io/${{ github.repository }}-dev
+    steps:
+      - name: Checkout Arrow
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        with:
+          fetch-depth: 0
+          submodules: recursive
+      - name: Cache Docker Volumes
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: .docker
+          key: jni-${{ matrix.platform.runs-on }}-${{ hashFiles('cpp/**') }}
+          restore-keys: jni-${{ matrix.platform.runs-on }}-
+      - name: Setup Python
+        uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0
+        with:
+          python-version: 3
+      - name: Setup Archery
+        run: python3 -m pip install -e dev/archery[docker]
+      - name: Execute Docker Build
+        env:
+          ARCHERY_DOCKER_USER: ${{ github.actor }}
+          ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,readwrite"
+        run: |
+          source ci/scripts/util_enable_core_dumps.sh
+          archery docker run cpp-jni
+      - name: Docker Push
+        if: >-
+          success() &&
+          github.event_name == 'push' &&
+          github.ref_name == 'main'
+        env:
+          ARCHERY_DOCKER_USER: ${{ github.actor }}
+          ARCHERY_DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        continue-on-error: true
+        run: archery docker push cpp-jni
+
   jni-macos:
     needs: check-labels
     name: JNI macOS
@@ -177,7 +238,7 @@ jobs:
       MACOSX_DEPLOYMENT_TARGET: "14.0"
     steps:
       - name: Checkout Arrow
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         with:
           fetch-depth: 0
           submodules: recursive

ci/docker/cpp-jni.dockerfile

@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base
+FROM ${base}
+
+ARG arch
+ARG arch_short
+
+SHELL ["/bin/bash", "-i", "-c"]
+ENTRYPOINT ["/bin/bash", "-i", "-c"]
+
+# Install basic dependencies
+RUN dnf install -y \
+    autoconf \
+    curl \
+    flex \
+    gdb \
+    git \
+    perl-IPC-Cmd \
+    wget \
+    zip
+
+# A system Python is required for Ninja and vcpkg in this Dockerfile.
+# On manylinux_2_28 base images, no system Python is installed.
+# We therefore override the PATH with Python 3.10 in /opt/python
+# so that we have a consistent Python version across base images.
+ENV CPYTHON_VERSION=cp310
+ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH}
+
+# Install CMake
+ARG cmake=3.29.2
+COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_cmake.sh ${cmake} /usr/local
+
+# Install Ninja
+ARG ninja=1.10.2
+COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local
+
+# Install ccache
+ARG ccache=4.1
+COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
+
+# Install vcpkg
+ARG vcpkg
+COPY ci/vcpkg/*.patch \
+     ci/vcpkg/*linux*.cmake \
+     ci/vcpkg/vcpkg.json \
+     arrow/ci/vcpkg/
+COPY ci/scripts/install_vcpkg.sh \
+     arrow/ci/scripts/
+ENV VCPKG_ROOT=/opt/vcpkg
+ARG build_type=release
+ENV CMAKE_BUILD_TYPE=${build_type} \
+    PATH="${PATH}:${VCPKG_ROOT}" \
+    VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
+    VCPKG_FEATURE_FLAGS="manifests" \
+    VCPKG_FORCE_SYSTEM_BINARIES=1 \
+    VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg
+# For --mount=type=secret: The GITHUB_TOKEN is the only real secret but we use
+# --mount=type=secret for GITHUB_REPOSITORY_OWNER and
+# VCPKG_BINARY_SOURCES too because we don't want to store them
+# into the built image in order to easily reuse the built image cache.
+#
+# For vcpkg install: cannot use the S3 feature here because while
+# aws-sdk-cpp=1.9.160 contains ssl related fixes as well as we can
+# patch the vcpkg portfile to support arm machines it hits ARROW-15141
+# where we would need to fall back to 1.8.186 but we cannot patch
+# those portfiles since vcpkg-tool handles the checkout of previous
+# versions => use bundled S3 build
+RUN --mount=type=secret,id=github_repository_owner \
+    --mount=type=secret,id=github_token \
+    --mount=type=secret,id=vcpkg_binary_sources \
+      export GITHUB_REPOSITORY_OWNER=$(cat /run/secrets/github_repository_owner); \
+      export GITHUB_TOKEN=$(cat /run/secrets/github_token); \
+      export VCPKG_BINARY_SOURCES=$(cat /run/secrets/vcpkg_binary_sources); \
+      arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg} && \
+      vcpkg install \
+        --clean-after-build \
+        --x-install-root=${VCPKG_ROOT}/installed \
+        --x-manifest-root=/arrow/ci/vcpkg \
+        --x-feature=azure \
+        --x-feature=dev \
+        --x-feature=flight \
+        --x-feature=gandiva \
+        --x-feature=gcs \
+        --x-feature=json \
+        --x-feature=orc \
+        --x-feature=parquet \
+        --x-feature=s3 && \
+      rm -rf ~/.config/NuGet/
+
+ENV ARROW_BUILD_TESTS=ON \
+    ARROW_CMAKE_ARGS="-DARROW_BUILD_TESTS=ON" \
+    CMAKE_PRESET=ninja-${CMAKE_BUILD_TYPE}-jni-linux

ci/scripts/cpp_build.sh

@@ -174,6 +174,11 @@ elif [ "${ARROW_EMSCRIPTEN:-OFF}" = "ON" ]; then
     -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
     ${ARROW_CMAKE_ARGS} \
     ${source_dir}
+elif [ -n "${CMAKE_PRESET}" ]; then
+  cmake \
+    --preset="${CMAKE_PRESET}" \
+    ${ARROW_CMAKE_ARGS} \
+    ${source_dir}
 else
   cmake \
     -Dabsl_SOURCE=${absl_SOURCE:-} \
@@ -308,10 +313,14 @@ fi
 popd
 
 if [ -x "$(command -v ldconfig)" ]; then
-  if [ -x "$(command -v sudo)" ]; then
-    SUDO=sudo
-  else
+  if [ "$(id --user)" -eq 0 ]; then
     SUDO=
+  else
+    if [ -x "$(command -v sudo)" ]; then
+      SUDO=sudo
+    else
+      SUDO=
+    fi
   fi
   ${SUDO} ldconfig ${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}
 fi

ci/scripts/cpp_test.sh

@@ -47,6 +47,12 @@ ctest_options=()
 if ! type azurite >/dev/null 2>&1; then
   exclude_tests+=("arrow-azurefs-test")
 fi
+if ! type storage-testbench >/dev/null 2>&1; then
+  exclude_tests+=("arrow-gcsfs-test")
+fi
+if ! type minio >/dev/null 2>&1; then
+  exclude_tests+=("arrow-s3fs-test")
+fi
 case "$(uname)" in
   Linux)
     n_jobs=$(nproc)
@@ -114,6 +120,41 @@ else
     "$@"
 fi
 
+# This is for testing find_package(Arrow).
+#
+# Note that this is not a perfect solution. We should improve this
+# later.
+#
+# * This is ad-hoc
+# * This doesn't test other CMake packages such as ArrowDataset
+if [ "${ARROW_USE_MESON:-OFF}" = "OFF" ] && \
+     [ "${ARROW_EMSCRIPTEN:-OFF}" = "OFF" ] && \
+     [ "${ARROW_USE_ASAN:-OFF}" = "OFF" ]; then
+  CMAKE_PREFIX_PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}"
+  case "$(uname)" in
+    MINGW*)
+      # <prefix>/lib/cmake/ isn't searched on Windows.
+      #
+      # See also:
+      # https://cmake.org/cmake/help/latest/command/find_package.html#config-mode-search-procedure
+      CMAKE_PREFIX_PATH+="/lib/cmake/"
+      ;;
+  esac
+  if [ -n "${VCPKG_ROOT}" ] && [ -n "${VCPKG_DEFAULT_TRIPLET}" ]; then
+    CMAKE_PREFIX_PATH+=";${VCPKG_ROOT}/installed/${VCPKG_DEFAULT_TRIPLET}"
+  fi
+  cmake \
+    -S "${source_dir}/examples/minimal_build" \
+    -B "${build_dir}/examples/minimal_build" \
+    -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
+  cmake --build "${build_dir}/examples/minimal_build"
+  pushd "${source_dir}/examples/minimal_build"
+  # PATH= is for Windows.
+  PATH="${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}}/bin:${PATH}" \
+    "${build_dir}/examples/minimal_build/arrow-example"
+  popd
+fi
+
 if [ "${ARROW_BUILD_EXAMPLES}" == "ON" ]; then
     examples=$(find "${binary_output_dir}" -executable -name "*example")
     if [ "${examples}" == "" ]; then

ci/vcpkg/vcpkg.json

@@ -19,6 +19,7 @@
     "re2",
     "snappy",
     "utf8proc",
+    "xsimd",
     "zlib",
     "zstd",
     {

cpp/CMakePresets.json

@@ -583,6 +583,37 @@
       "displayName": "Benchmarking build with everything enabled",
       "cacheVariables": {}
     },
+    {
+      "name": "ninja-release-jni-linux",
+      "inherits": [
+        "base-release"
+      ],
+      "displayName": "Build for JNI on Linux",
+      "cacheVariables": {
+        "ARROW_ACERO": "ON",
+        "ARROW_BUILD_SHARED": "OFF",
+        "ARROW_BUILD_STATIC": "ON",
+        "ARROW_CSV": "ON",
+        "ARROW_DATASET": "ON",
+        "ARROW_DEPENDENCY_SOURCE": "VCPKG",
+        "ARROW_DEPENDENCY_USE_SHARED": "OFF",
+        "ARROW_GANDIVA": "ON",
+        "ARROW_GANDIVA_STATIC_LIBSTDCPP": "ON",
+        "ARROW_GCS": "ON",
+        "ARROW_JSON": "ON",
+        "ARROW_ORC": "ON",
+        "ARROW_PARQUET": "ON",
+        "ARROW_RPATH_ORIGIN": "ON",
+        "ARROW_S3": "ON",
+        "ARROW_SUBSTRAIT": "ON",
+        "PARQUET_BUILD_EXAMPLES": "OFF",
+        "PARQUET_BUILD_EXECUTABLES": "OFF",
+        "PARQUET_REQUIRE_ENCRYPTION": "OFF",
+        "VCPKG_MANIFEST_MODE": "OFF",
+        "VCPKG_ROOT": "$env{VCPKG_ROOT}",
+        "VCPKG_TARGET_TRIPLET": "$env{VCPKG_TARGET_TRIPLET}"
+      }
+    },
     {
       "name": "ninja-release-jni-macos",
       "inherits": [

cpp/src/arrow/testing/process.cc

@@ -176,7 +176,7 @@ class Process::Impl {
       for (const auto& kv : process::environment::current()) {
         env[kv.key()] = process::environment::value(kv.value());
       }
-      env["PATH"] = process::environment::value(current_exe.parent_path());
+      env["PATH"] = process::environment::value(current_exe.parent_path().string());
       executable_ = process::environment::find_executable(name, env);
 #  else
       executable_ = process::search_path(name, {current_exe.parent_path()});

cpp/src/gandiva/tests/projector_test.cc

@@ -389,6 +389,10 @@ TEST_F(TestProjector, TestAllIntTypes) {
 }
 
 TEST_F(TestProjector, TestExtendedMath) {
+#ifdef __aarch64__
+  GTEST_SKIP() << "Failed on aarch64 with 'JIT session error: Symbols not found: [ "
+                  "__multf3, __subtf3, __trunctfdf2, __extenddftf2, __divtf3 ]'";
+#endif
   // schema for input fields
   auto field0 = arrow::field("f0", arrow::float64());
   auto field1 = arrow::field("f1", arrow::float64());