Arrow with a python feature

.github/workflows/macos.yml

@@ -122,7 +122,6 @@ jobs:
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-bottleneck
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-numexpr
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-calamine
-          ./vcpkg/vcpkg install $VCPKG_OPTIONS py-pyarrow
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-adbc-driver-manager
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-adbc-postgresql
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-adbc-sqlite
@@ -132,8 +131,8 @@ jobs:
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-scipy
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-pandas
           ./vcpkg/vcpkg install $VCPKG_OPTIONS py-pyogrio
-          ./vcpkg/vcpkg install $VCPKG_OPTIONS 'py-geopandas[arrow]'
-
+          ./vcpkg/vcpkg install $VCPKG_OPTIONS py-geopandas
+          ./vcpkg/vcpkg install $VCPKG_OPTIONS arrow[core,csv,filesystem,json,parquet,python] # defaults + python
           # ./vcpkg/vcpkg install $VCPKG_OPTIONS py-pyqt6
 
       - name: 📑 Upload logs

ports/arrow/0001-msvc-static-name.patch

@@ -0,0 +1,13 @@
+diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
+index 391c43e0a..50f6d3d3c 100644
+--- a/cpp/cmake_modules/BuildUtils.cmake
++++ b/cpp/cmake_modules/BuildUtils.cmake
+@@ -427,7 +427,7 @@ function(ADD_ARROW_LIB LIB_NAME)
+       target_include_directories(${LIB_NAME}_static PRIVATE ${ARG_PRIVATE_INCLUDES})
+     endif()
+
+-    if(MSVC_TOOLCHAIN)
++    if(MSVC_TOOLCHAIN AND 0)
+       set(LIB_NAME_STATIC ${LIB_NAME}_static)
+     else()
+       set(LIB_NAME_STATIC ${LIB_NAME})

ports/arrow/0003-android-musl.patch

@@ -0,0 +1,14 @@
+diff --git a/cpp/src/arrow/vendored/musl/strptime.c b/cpp/src/arrow/vendored/musl/strptime.c
+index 41912fd..0ea36e9 100644
+--- a/cpp/src/arrow/vendored/musl/strptime.c
++++ b/cpp/src/arrow/vendored/musl/strptime.c
+@@ -18,7 +18,9 @@
+ #undef HAVE_LANGINFO
+
+ #ifndef _WIN32
++# if !(defined(__ANDROID__) && __ANDROID_API__ < 26)
+ #define HAVE_LANGINFO 1
++# endif
+ #endif
+
+ #ifdef HAVE_LANGINFO

ports/arrow/0004-android-datetime.patch

@@ -0,0 +1,29 @@
+diff --git a/cpp/src/arrow/vendored/datetime/tz.h b/cpp/src/arrow/vendored/datetime/tz.h
+index 61ab3df106..d456d6765f 100644
+--- a/cpp/src/arrow/vendored/datetime/tz.h
++++ b/cpp/src/arrow/vendored/datetime/tz.h
+@@ -858,7 +858,9 @@ private:
+     load_data(std::istream& inf, std::int32_t tzh_leapcnt, std::int32_t tzh_timecnt,
+                                  std::int32_t tzh_typecnt, std::int32_t tzh_charcnt);
+ # if defined(ANDROID) || defined(__ANDROID__)
++public:
+     void parse_from_android_tzdata(std::ifstream& inf, const std::size_t off);
++private:
+ # endif // defined(ANDROID) || defined(__ANDROID__)
+ #else  // !USE_OS_TZDB
+     DATE_API sys_info   get_info_impl(sys_seconds tp, int tz_int) const;
+diff --git a/cpp/src/arrow/vendored/datetime/visibility.h b/cpp/src/arrow/vendored/datetime/visibility.h
+index 780c00d70b..a9514edba7 100644
+--- a/cpp/src/arrow/vendored/datetime/visibility.h
++++ b/cpp/src/arrow/vendored/datetime/visibility.h
+@@ -21,6 +21,10 @@
+ #  define USE_OS_TZDB 1
+ #endif
+
++#if defined(ANDROID) || defined(__ANDROID__)
++#  define BUILD_TZ_LIB
++#endif
++
+ #if defined(ARROW_STATIC)
+ // intentially empty
+ #elif defined(ARROW_EXPORTING)

ports/arrow/0005-cmake-msvcruntime.patch

@@ -0,0 +1,24 @@
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+index abfe6d274f..8bacfe89af 100644
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -886,9 +886,17 @@ foreach(CONFIG DEBUG MINSIZEREL RELEASE RELWITHDEBINFO)
+   set(EP_CXX_FLAGS_${CONFIG} "${CMAKE_CXX_FLAGS_${CONFIG}}")
+   set(EP_C_FLAGS_${CONFIG} "${CMAKE_C_FLAGS_${CONFIG}}")
+   if(CONFIG STREQUAL DEBUG)
+-    set(EP_MSVC_RUNTIME_LIBRARY MultiThreadedDebugDLL)
++    if(BUILD_SHARED_LIBS)
++      set(EP_MSVC_RUNTIME_LIBRARY MultiThreadedDebugDLL)
++     else()
++      set(EP_MSVC_RUNTIME_LIBRARY MultiThreadedDebug)
++    endif()
+   else()
+-    set(EP_MSVC_RUNTIME_LIBRARY MultiThreadedDLL)
++    if(BUILD_SHARED_LIBS)
++      set(EP_MSVC_RUNTIME_LIBRARY MultiThreadedDLL)
++    else()
++      set(EP_MSVC_RUNTIME_LIBRARY MultiThreaded)
++    endif()
+   endif()
+   string(APPEND EP_CXX_FLAGS_${CONFIG}
+          " ${CMAKE_CXX_COMPILE_OPTIONS_MSVC_RUNTIME_LIBRARY_${EP_MSVC_RUNTIME_LIBRARY}}")

ports/arrow/0006-pcg-msvc-arm64.patch

@@ -0,0 +1,43 @@
+diff --git a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp
+index 0181e69e4e..012f3d6682 100644
+--- a/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp
++++ b/cpp/src/arrow/vendored/pcg/pcg_uint128.hpp
+@@ -67,7 +67,7 @@
+         #define PCG_LITTLE_ENDIAN 1
+     #elif __BIG_ENDIAN__ || _BIG_ENDIAN
+         #define PCG_LITTLE_ENDIAN 0
+-    #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86
++    #elif __x86_64 || __x86_64__ || _M_X64 || __i386 || __i386__ || _M_IX86 || _M_ARM64
+         #define PCG_LITTLE_ENDIAN 1
+     #elif __powerpc__ || __POWERPC__ || __ppc__ || __PPC__ \
+           || __m68k__ || __mc68000__
+@@ -734,7 +734,13 @@ uint_x4<UInt,UIntX2> operator*(const uint_x4<UInt,UIntX2>& a,
+
+ #if PCG_64BIT_SPECIALIZATIONS
+ #if defined(_MSC_VER)
++#if defined(_M_X64) || defined(_M_IX86)
+ #pragma intrinsic(_umul128)
++#elif defined(_M_ARM64)
++#pragma intrinsic(__umulh)
++#else
++#error Unsupported architecture
++#endif
+ #endif
+
+ #if defined(_MSC_VER) || __SIZEOF_INT128__
+@@ -743,8 +749,15 @@ uint_x4<UInt32,uint64_t> operator*(const uint_x4<UInt32,uint64_t>& a,
+ 				   const uint_x4<UInt32,uint64_t>& b)
+ {
+ #if defined(_MSC_VER)
++#if defined(_M_X64) || defined(_M_IX86)
+     uint64_t hi;
+     uint64_t lo = _umul128(a.d.v01, b.d.v01, &hi);
++#elif defined(_M_ARM64)
++    uint64_t lo = a.d.v01 * b.d.v01;
++    uint64_t hi = __umulh(a.d.v01, b.d.v01);
++#else
++#error Unsupported architecture
++#endif
+ #else
+     __uint128_t r = __uint128_t(a.d.v01) * __uint128_t(b.d.v01);
+     uint64_t lo = uint64_t(r);

ports/arrow/portfile.cmake

@@ -0,0 +1,219 @@
+vcpkg_download_distfile(
+    ARCHIVE_PATH
+    URLS "https://archive.apache.org/dist/arrow/arrow-${VERSION}/apache-arrow-${VERSION}.tar.gz"
+    FILENAME apache-arrow-${VERSION}.tar.gz
+    SHA512 8ec9ddaf7917c0e35c8bb32831fe6ea0a7b81de5723828a1289ba1b9e104b42af688d0f427a0ceff6f617d5f7ac67769431184b137e54f6987779e467c59d3ec
+)
+vcpkg_extract_source_archive(
+    SOURCE_PATH
+    ARCHIVE ${ARCHIVE_PATH}
+    PATCHES
+        0001-msvc-static-name.patch
+        0003-android-musl.patch
+        0004-android-datetime.patch
+        0005-cmake-msvcruntime.patch
+        0006-pcg-msvc-arm64.patch
+)
+
+# Check cpp/cmake_modules/DefineOptions.cmake for option dependencies -
+# they must be modeled as feature dependencies in vcpkg.json.
+vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+    FEATURES
+        acero       ARROW_ACERO
+        compute     ARROW_COMPUTE
+        csv         ARROW_CSV
+        cuda        ARROW_CUDA
+        dataset     ARROW_DATASET
+        filesystem  ARROW_FILESYSTEM
+        flight      ARROW_FLIGHT
+        flightsql   ARROW_FLIGHT_SQL
+        gcs         ARROW_GCS
+        jemalloc    ARROW_JEMALLOC
+        json        ARROW_JSON
+        mimalloc    ARROW_MIMALLOC
+        orc         ARROW_ORC
+        parquet     ARROW_PARQUET
+        parquet     PARQUET_REQUIRE_ENCRYPTION
+        s3          ARROW_S3
+)
+
+if(VCPKG_TARGET_IS_WINDOWS AND NOT VCPKG_TARGET_IS_MINGW)
+    list(APPEND FEATURE_OPTIONS "-DARROW_USE_NATIVE_INT128=OFF")
+endif()
+
+if(VCPKG_TARGET_IS_WINDOWS AND VCPKG_TARGET_ARCHITECTURE STREQUAL "arm64")
+    list(APPEND FEATURE_OPTIONS "-DARROW_SIMD_LEVEL=NONE")
+endif()
+
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_BUILD_SHARED)
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "static" ARROW_BUILD_STATIC)
+string(COMPARE EQUAL ${VCPKG_LIBRARY_LINKAGE} "dynamic" ARROW_DEPENDENCY_USE_SHARED)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}/cpp"
+    OPTIONS
+        ${FEATURE_OPTIONS}
+        -DARROW_BUILD_SHARED=${ARROW_BUILD_SHARED}
+        -DARROW_BUILD_STATIC=${ARROW_BUILD_STATIC}
+        -DARROW_BUILD_TESTS=OFF
+        -DARROW_DEPENDENCY_SOURCE=SYSTEM
+        -DARROW_DEPENDENCY_USE_SHARED=${ARROW_DEPENDENCY_USE_SHARED}
+        -DARROW_PACKAGE_KIND=vcpkg
+        -DARROW_WITH_BROTLI=ON
+        -DARROW_WITH_BZ2=ON
+        -DARROW_WITH_LZ4=ON
+        -DARROW_WITH_SNAPPY=ON
+        -DARROW_WITH_ZLIB=ON
+        -DARROW_WITH_ZSTD=ON
+        -DBUILD_WARNING_LEVEL=PRODUCTION
+        -DZSTD_MSVC_LIB_PREFIX=
+    MAYBE_UNUSED_VARIABLES
+        ZSTD_MSVC_LIB_PREFIX
+)
+
+vcpkg_cmake_install()
+vcpkg_copy_pdbs()
+
+vcpkg_fixup_pkgconfig()
+
+if(EXISTS "${CURRENT_PACKAGES_DIR}/lib/arrow_static.lib")
+    message(FATAL_ERROR "Installed lib file should be named 'arrow.lib' via patching the upstream build.")
+endif()
+
+if("dataset" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME arrowdataset
+        CONFIG_PATH lib/cmake/ArrowDataset
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+if("acero" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME arrowacero
+        CONFIG_PATH lib/cmake/ArrowAcero
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+if("compute" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME arrowcompute
+        CONFIG_PATH lib/cmake/ArrowCompute
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+if("flight" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME arrowflight
+        CONFIG_PATH lib/cmake/ArrowFlight
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+if("flightsql" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME arrowflightsql
+        CONFIG_PATH lib/cmake/ArrowFlightSql
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+if("parquet" IN_LIST FEATURES)
+    vcpkg_cmake_config_fixup(
+        PACKAGE_NAME parquet
+        CONFIG_PATH lib/cmake/Parquet
+        DO_NOT_DELETE_PARENT_CONFIG_PATH
+    )
+endif()
+
+file(GLOB main_configs "${CURRENT_PACKAGES_DIR}/lib/cmake/Arrow/*onfig.cmake")
+file(GLOB extra_configs "${CURRENT_PACKAGES_DIR}/lib/cmake/*/*onfig.cmake")
+list(REMOVE_ITEM extra_configs ${main_configs})
+if(NOT "${extra_configs}" STREQUAL "")
+    message("${Z_VCPKG_BACKCOMPAT_MESSAGE_LEVEL}"
+        "Unhandled CMake config: ${extra_configs}\n"
+        "This might be caused by insufficient feature dependencies in ports/arrow/vcpkg.json."
+    )
+endif()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/Arrow)
+
+file(INSTALL "${CMAKE_CURRENT_LIST_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}")
+foreach(feature IN ITEMS parquet dataset acero compute flight flightsql)
+    if(feature IN_LIST FEATURES)
+        file(READ "${CMAKE_CURRENT_LIST_DIR}/usage-${feature}" feature_usage)
+        file(APPEND "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" "${feature_usage}")
+    endif()
+endforeach()
+
+if("example" IN_LIST FEATURES)
+    file(INSTALL "${SOURCE_PATH}/cpp/examples/minimal_build/" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}/example")
+endif()
+
+if("python" IN_LIST FEATURES)
+    # use the vcpkg-installed python so we create the wheel for the correct version
+    message(STATUS "Building pyarrow")
+
+    set(PYTHON3 "${CURRENT_HOST_INSTALLED_DIR}/tools/python3/python3${VCPKG_HOST_EXECUTABLE_SUFFIX}")
+    if(NOT EXISTS "${PYTHON3}")
+        vcpkg_find_acquire_program(PYTHON3)
+    endif()
+    x_vcpkg_get_python_packages(
+        PYTHON_VERSION 3
+        PYTHON_EXECUTABLE "${PYTHON3}"
+        REQUIREMENTS_FILE "${SOURCE_PATH}/python/requirements-build.txt"
+        OUT_PYTHON_VAR PYTHON3_VENV
+    )
+
+    set(ENV{Arrow_DIR} "${CURRENT_PACKAGES_DIR}/share/arrow")
+    set(ENV{ArrowCompute_DIR} "${CURRENT_PACKAGES_DIR}/share/arrowcompute")
+    if("dataset" IN_LIST FEATURES)
+        set(ENV{ArrowDataset_DIR} "${CURRENT_PACKAGES_DIR}/share/arrowdataset")
+    endif()
+    if("parquet" IN_LIST FEATURES)
+        set(ENV{Parquet_DIR} "${CURRENT_PACKAGES_DIR}/share/parquet")
+    endif()
+    if("acero" IN_LIST FEATURES)
+        set(ENV{ArrowAcero_DIR} "${CURRENT_PACKAGES_DIR}/share/arrowacero")
+    endif()
+    if("flight" IN_LIST FEATURES)
+        set(ENV{ArrowFlight_DIR} "${CURRENT_PACKAGES_DIR}/share/arrowflight")
+    endif()
+    if ("flightsql" IN_LIST FEATURES)
+         set(ENV{ArrowFlightSql_DIR} "${CURRENT_PACKAGES_DIR}/share/arrowflightsql")
+    endif()
+
+    set(ENV{SETUPTOOLS_SCM_PRETEND_VERSION} "${VERSION}")
+    set(ENV{PDM_BUILD_SCM_VERSION} "${VERSION}")
+
+    set(install_prefix "${CURRENT_INSTALLED_DIR}")
+    if(VCPKG_TARGET_IS_WINDOWS)
+        string(APPEND install_prefix "/tools/python3")
+    endif()
+
+    if (NOT "${VCPKG_BUILD_TYPE}" STREQUAL "")
+        set(build_opts "--build-type=${VCPKG_BUILD_TYPE}")
+    else()
+        set(build_opts "--build-type=release")
+    endif()
+    vcpkg_execute_required_process(
+        COMMAND "${PYTHON3_VENV}" "setup.py"  
+        "build_ext" "${build_opts}" "--bundle-arrow-cpp"
+        "install" "--prefix" "${install_prefix}"
+        LOGNAME "python-build-${TARGET_TRIPLET}"
+        WORKING_DIRECTORY "${SOURCE_PATH}/python"
+    )
+
+    message(STATUS "Testing pyarrow")
+    vcpkg_execute_required_process(COMMAND "${PYTHON3}" "${SOURCE_PATH}/python/scripts/test_imports.py"
+        LOGNAME "python-import-test-${TARGET_TRIPLET}"
+        WORKING_DIRECTORY "${SOURCE_PATH}/python"
+    )
+endif()
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share/doc")
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.txt")

ports/arrow/usage

@@ -0,0 +1,4 @@
+The package arrow provides CMake targets:
+
+    find_package(Arrow CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Arrow::arrow_static,Arrow::arrow_shared>")

ports/arrow/usage-acero

@@ -0,0 +1,3 @@
+
+    find_package(ArrowAcero CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowAcero::arrow_acero_static,ArrowAcero::arrow_acero_shared>")

ports/arrow/usage-compute

@@ -0,0 +1,3 @@
+
+    find_package(ArrowCompute CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowCompute::arrow_compute_static,ArrowCompute::arrow_compute_shared>")

ports/arrow/usage-dataset

@@ -0,0 +1,3 @@
+
+    find_package(ArrowDataset CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowDataset::arrow_dataset_static,ArrowDataset::arrow_dataset_shared>")

ports/arrow/usage-flight

@@ -0,0 +1,3 @@
+
+    find_package(ArrowFlight CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowFlight::arrow_flight_static,ArrowFlight::arrow_flight_shared>")

ports/arrow/usage-flightsql

@@ -0,0 +1,3 @@
+
+    find_package(ArrowFlightSql CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,ArrowFlightSql::arrow_flight_sql_static,ArrowFlightSql::arrow_flight_sql_shared>")

ports/arrow/usage-parquet

@@ -0,0 +1,3 @@
+
+    find_package(Parquet CONFIG REQUIRED)
+    target_link_libraries(main PRIVATE "$<IF:$<BOOL:${ARROW_BUILD_STATIC}>,Parquet::parquet_static,Parquet::parquet_shared>")