update torchcodec in ci (#7764)

* update torchcodec in ci

* ffmpeg for windows ci

* fix tests

Author: lhoestq

.github/workflows/ci.yml

@@ -53,6 +53,17 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.9"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.9"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
       - name: Install uv
@@ -86,11 +97,22 @@ jobs:
         if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           sudo apt update
-          sudo apt install -y ffmpeg 
+          sudo apt install -y ffmpeg
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.11"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
       - name: Install uv
@@ -123,6 +145,17 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.11"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
       - name: Upgrade pip
         run: python -m pip install --upgrade pip
       - name: Install uv

setup.py

@@ -126,7 +126,7 @@
     "multiprocess<0.70.17",  # to align with dill<0.3.9 (see above)
     # to save datasets locally or on any filesystem
     # minimum 2023.1.0 to support protocol=kwargs in fsspec's `open`, `get_fs_token_paths`, etc.: see https://github.com/fsspec/filesystem_spec/pull/1143
-    "fsspec[http]>=2023.1.0,<=2025.7.0",
+    "fsspec[http]>=2023.1.0,<=2025.9.0",
     # To get datasets from the Datasets Hub on huggingface.co
     "huggingface-hub>=0.24.0",
     # Utilities from PyPA to e.g., compare versions
@@ -184,7 +184,7 @@
     "zstandard",
     "polars[timezone]>=0.20.0",
     "Pillow>=9.4.0",  # When PIL.Image.ExifTags was introduced
-    "torchcodec>=0.6.0; sys_platform != 'win32'",  # not available for windows
+    "torchcodec>=0.7.0",  # minium version to get windows support
 ]
 
 NUMPY2_INCOMPATIBLE_LIBRARIES = [

src/datasets/arrow_dataset.py

@@ -1549,9 +1549,6 @@ def save_to_disk(
             num_shards = int(dataset_nbytes / max_shard_size) + 1
             num_shards = max(num_shards, num_proc or 1)
 
-        num_proc = num_proc if num_proc is not None else 1
-        num_shards = num_shards if num_shards is not None else num_proc
-
         fs: fsspec.AbstractFileSystem
         fs, _ = url_to_fs(dataset_path, **(storage_options or {}))
 
@@ -1609,7 +1606,7 @@ def save_to_disk(
         )
         shard_lengths = [None] * num_shards
         shard_sizes = [None] * num_shards
-        if num_proc >= 1:
+        if num_proc is not None and num_proc >= 1:
             with Pool(num_proc) as pool:
                 with pbar:
                     for job_id, done, content in iflatmap_unordered(