Test windows build

.bazelrc

@@ -29,6 +29,7 @@ build:linux --force_pic
 build:macos --force_pic
 build:clang-cl --compiler=clang-cl
 build:msvc --compiler=msvc-cl
+build:windows --compiler=clang-cl
 
 # test config
 test --build_tests_only

.github/workflows/ci.yml

@@ -39,215 +39,21 @@ on:
     types: ["opened", "reopened", "synchronize"]
 
 jobs:
-  java:
-    name: Java CI
-    runs-on: ubuntu-latest
-    env:
-      MY_VAR: "PATH"
-    strategy:
-      matrix:
-        java-version: ["8", "11", "17", "21"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK ${{ matrix.java-version }}
-        uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: "temurin"
-      - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
-      - name: Install python
-        run: ./ci/run_ci.sh install_python
-      - name: Install pyfury
-        run: ./ci/run_ci.sh install_pyfury
-      - name: Run CI with Maven
-        run: ./ci/run_ci.sh java${{ matrix.java-version }}
-  openj9:
-    name: Openj9 Java CI
-    runs-on: ubuntu-latest
-    env:
-      MY_VAR: "PATH"
-    strategy:
-      matrix:
-        # String in openj9 1.8 share byte array by offset, fury doesn't allow it.
-        java-version: ["21"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK ${{ matrix.java-version }}
-        uses: actions/setup-java@v4
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: "adopt-openj9"
-      - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
-      - name: Install python
-        run: ./ci/run_ci.sh install_python
-      - name: Install pyfury
-        run: ./ci/run_ci.sh install_pyfury
-      - name: Run CI with Maven
-        run: ./ci/run_ci.sh java${{ matrix.java-version }}
-  graalvm:
-    name: GraalVM CI
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        java-version: ["17", "21", "22"]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: graalvm/setup-graalvm@v1
-        with:
-          java-version: ${{ matrix.java-version }}
-          distribution: "graalvm"
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          native-image-job-reports: "true"
-      - name: Build native image and run
-        run: ./ci/run_ci.sh graalvm_test
-
-  scala:
-    name: Scala CI
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK8
-        uses: actions/setup-java@v4
-        with:
-          java-version: 8
-          distribution: "temurin"
-      - name: Install fury java
-        run: cd java && mvn -T10 --no-transfer-progress clean install -DskipTests && cd -
-      - name: Test
-        run: |
-          # Avoid sbt download jackson-databind error
-          rm -rf /home/runner/.m2/repository/com/fasterxml/jackson/
-          cd scala && sbt +test && cd -
-
-  integration_tests:
-    name: Integration Tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK8
-        uses: actions/setup-java@v4
-        with:
-          java-version: 8
-          distribution: "temurin"
-      - name: Run CI
-        run: ./ci/run_ci.sh integration_tests
-
-  javascript:
-    name: JavaScript CI
-    strategy:
-      matrix:
-        node-version: [14.x, 16.x, 18.x, 20.x]
-        os: [ubuntu-latest, macos-13, windows-2022]
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v4
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Upgrade npm
-        run: npm install -g npm@8
-      # node-gyp needs to use python and relies on the distutils module.
-      # The distutils module has been removed starting from python 3.12
-      # (see https://docs.python.org/3.10/library/distutils.html). Some
-      # OS (such as macos -latest) uses python3.12 by default, so python 3.8
-      # is used here to avoid this problem.
-      - name: Set up Python3.8
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - name: Run CI with NodeJS
-        run: python ./ci/run_ci.py javascript
-
-  rust:
-    name: Rust CI
-    strategy:
-      matrix:
-        os: [ubuntu-latest, macos-12, macos-14] # macos-12: x86, macos-14: arm64
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 45
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.11
-      - name: Run Rust CI
-        run: python ./ci/run_ci.py rust
 
   cpp:
     name: C++ CI
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-12, macos-14] # macos-12: x86, macos-14: arm64
+        os: [ubuntu-latest, windows-2022] # macos-12: x86, macos-14: arm64
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: 3.11
+      - uses: actions/checkout@v4
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
       - name: Run C++ CI with Bazel
         run: python ./ci/run_ci.py cpp
-  python:
-    name: Python CI
-    # Fix python 3.6 install issue, see
-    # https://github.com/rwth-i6/returnn/commit/38ecab17d781c4b74db6a174c8097187380b4ddc
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [3.7, 3.12]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
-      - name: Run Python CI
-        run: ./ci/run_ci.sh python
-
-  go:
-    name: Golang CI
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        go-version: ["1.13", "1.18"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Go ${{ matrix.go-version }}
-        uses: actions/setup-go@v4
-        with:
-          go-version: ${{ matrix.go-version }}
-      - name: Display Go version
-        run: go version
-      - name: Install bazel
-        run: ./ci/run_ci.sh install_bazel
-      - name: Install python
-        run: ./ci/run_ci.sh install_python
-      - name: Install pyfury
-        run: ./ci/run_ci.sh install_pyfury
-      - name: Run Golang CI
-        run: ./ci/run_ci.sh go
-
-  lint:
-    name: Code Style Check
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up JDK ${{ matrix.java-version }}
-        uses: actions/setup-java@v4
-        with:
-          java-version: 21
-          distribution: "oracle"
-      - name: Check License Header
-        uses: korandoru/hawkeye@v3
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.8
-      - name: Check code style
-        run: ./ci/run_ci.sh format

bazel/arrow/pyarrow_configure.bzl

@@ -172,12 +172,15 @@ def _get_pyarrow_include(repository_ctx, python_bin="python3"):
         error_details=(
             "Is the Python binary path set up right? " + "(See ./configure or "
             + python_bin + ".) " + "Is distutils installed?"))
-    return result.stdout.splitlines()[0]
+    return result.stdout.splitlines()[0].replace('\\', '/')
 
 def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python3"):
     """Gets the pyarrow shared library path."""
+    if not _is_windows(repository_ctx):
+        library_name = "lib" + library_name
+
     code = """import pyarrow, os, glob;print(glob.glob(os.path.join(""" +\
-        """os.path.dirname(pyarrow.__file__), 'lib{}.*'))[0])""".format(library_name)
+        """os.path.dirname(pyarrow.__file__), '{}.*'))[0])""".format(library_name)
     result = _execute(
         repository_ctx, [
             python_bin, "-c", code
@@ -186,7 +189,7 @@ def _get_pyarrow_shared_library(repository_ctx, library_name, python_bin="python
         error_details=(
             "Is the Python binary path set up right? " + "(See ./configure or "
             + python_bin + ".) " + "Is distutils installed?"))
-    return result.stdout.splitlines()[0]
+    return result.stdout.splitlines()[0].replace('\\', '/')
 
 #python numpy include
 def _get_python_numpy_include(repository_ctx, python_bin="python3"):
@@ -199,29 +202,35 @@ def _get_python_numpy_include(repository_ctx, python_bin="python3"):
         error_details=(
             "Is the Python binary path set up right? " + "(See ./configure or "
             + python_bin + ".) " + "Is distutils installed?"))
-    return result.stdout.splitlines()[0]
+    return result.stdout.splitlines()[0].replace('\\', '/')
 
 def _pyarrow_pip_impl(repository_ctx):
-    arrow_header_dir = _get_pyarrow_include(repository_ctx)
+    python_bin = "python3"
+
+    # python 3.x is usually named as `python` by default on windows.
+    if _is_windows(repository_ctx):
+        python_bin = "python"
+
+    arrow_header_dir = _get_pyarrow_include(repository_ctx, python_bin)
     arrow_header_rule = _symlink_genrule_for_dir(
         repository_ctx,
         arrow_header_dir,
         "include",
         "arrow_header_include",
     )
 
-    arrow_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow")
+    arrow_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow", python_bin)
     arrow_library = arrow_library_path.rsplit("/",1 )[-1]
     arrow_library_rule = _symlink_genrule_for_dir(
         repository_ctx, None, "", "libarrow", [arrow_library_path], [arrow_library])
 
-    arrow_python_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow_python")
+    arrow_python_library_path = _get_pyarrow_shared_library(repository_ctx, "arrow_python", python_bin)
     arrow_python_library = arrow_python_library_path.rsplit("/",1 )[-1]
     arrow_python_library_rule = _symlink_genrule_for_dir(
         repository_ctx, None, "", "libarrow_python",
         [arrow_python_library_path], [arrow_python_library])
 
-    python_numpy_include = _get_python_numpy_include(repository_ctx)
+    python_numpy_include = _get_python_numpy_include(repository_ctx, python_bin)
     python_numpy_include_rule = _symlink_genrule_for_dir(
         repository_ctx, python_numpy_include, 'python_numpy_include', 'python_numpy_include')
 

ci/run_ci.py

@@ -28,6 +28,8 @@
 
 PYARROW_VERSION = "14.0.0"
 
+NUMPY_VERSION = "1.26.4"
+
 PROJECT_ROOT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../")
 
 logging.basicConfig(
@@ -115,7 +117,7 @@ def _run_js():
 def _install_cpp_deps():
     _exec_cmd(f"pip install pyarrow=={PYARROW_VERSION}")
     _exec_cmd("pip install psutil")
-    _exec_cmd("pip install 'numpy<2.0.0'")
+    _exec_cmd(f"pip install numpy=={NUMPY_VERSION}")
     _install_bazel()
 
 

cpp/fury/meta/preprocessor.h

@@ -25,16 +25,26 @@
 // passed to the macros are fully expanded before they are concatenated.
 #define FURY_PP_CONCAT(a, b) FURY_PP_CONCAT_IMPL(a, b)
 
-#define FURY_PP_NARG_IMPL(...) FURY_PP_NARG_CALC(__VA_ARGS__)
-#define FURY_PP_NARG(...) FURY_PP_NARG_IMPL(__VA_ARGS__, FURY_PP_NARG_REV())
+#define FURY_PP_EXPAND(x) x
 
-#define FURY_PP_NARG_CALC(                                                     \
-    _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16,     \
-    _17, _18, _19, _20, _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, _31, \
-    _32, _33, _34, _35, _36, _37, _38, _39, _40, _41, _42, _43, _44, _45, _46, \
-    _47, _48, _49, _50, _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, _61, \
-    _62, _63, N, ...)                                                          \
-  N
+#define FURY_PP_NARG_IMPL(...) \
+    FURY_PP_EXPAND(FURY_PP_NARG_CALC(__VA_ARGS__, \
+        63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, \
+        49, 48, 47, 46, 45, 44, 43, 42, 41, 40, 39, 38, 37, 36, \
+        35, 34, 33, 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, \
+        21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, \
+        7, 6, 5, 4, 3, 2, 1, 0))
+
+#define FURY_PP_NARG_CALC( \
+    _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, \
+    _11, _12, _13, _14, _15, _16, _17, _18, _19, _20, \
+    _21, _22, _23, _24, _25, _26, _27, _28, _29, _30, \
+    _31, _32, _33, _34, _35, _36, _37, _38, _39, _40, \
+    _41, _42, _43, _44, _45, _46, _47, _48, _49, _50, \
+    _51, _52, _53, _54, _55, _56, _57, _58, _59, _60, \
+    _61, _62, _63, N, ...) N
+
+#define FURY_PP_NARG(...) FURY_PP_NARG_IMPL(__VA_ARGS__)
 
 #define FURY_PP_NARG_REV()                                                     \
   63, 62, 61, 60, 59, 58, 57, 56, 55, 54, 53, 52, 51, 50, 49, 48, 47, 46, 45,  \

cpp/fury/row/row.cc

@@ -237,8 +237,8 @@ int *ArrayData::GetDimensions(ArrayData &array, int num_dims) {
   // use deep-first search to search to numDimensions-1 layer to get dimensions.
   int depth = 0;
   auto dimensions = new int[num_dims];
-  int start_from_lefts[num_dims];
-  ArrayData *arrs[num_dims]; // root to current node
+  std::vector<int> start_from_lefts(num_dims);
+  std::vector<const ArrayData *> arrs(num_dims); // root to current node
   ArrayData &arr = array;
   while (depth < num_dims) {
     arrs[depth] = &arr;

cpp/fury/thirdparty/MurmurHash3.cc

@@ -15,6 +15,7 @@
 // Microsoft Visual Studio
 
 #if defined(_MSC_VER)
+#include <intrin.h>
 
 #define FORCE_INLINE __forceinline