Merge current upstream dev into PR 9140 observation branch

Integrate upstream dev at 9a6ac14 without conflicts. Preserve the Python 3.12 dependency-resolution workflow byte for byte and retain every existing upstream gate.

Validation: Ruby/Psych static YAML comparison, observational-job configuration assertions, bash -n for job run blocks, and git diff --cached --check. No Python, dependency installs, MONAI tests or Actions execution.

Assisted-by: OpenAI Codex <noreply@openai.com>
Signed-off-by: Demetrios Agourakis <demetrios@agourakis.med.br>

Author: agourakis82

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
       - id: mixed-line-ending
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.16.5
+    rev: v0.16.10
     hooks:
     -   id: ruff-check
         args: ["--fix"]
@@ -43,7 +43,7 @@ repos:
           )
 
   - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 26.5.1  # Black version, keep synced with MONAI requirements
+    rev: 26.10.0  # Black version, keep synced with MONAI requirements
     hooks:
       - id: black
         language_version: python3
@@ -55,7 +55,7 @@ repos:
           )
 
   - repo: https://github.com/pycqa/isort
-    rev: 9.0.1  # isort version, keep synced with MONAI requirements
+    rev: 9.0.2  # isort version, keep synced with MONAI requirements
     hooks:
       - id: isort
         name: isort (python)

Dockerfile.rocm

@@ -0,0 +1,126 @@
+# Copyright (c) MONAI Consortium
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# MONAI on AMD ROCm (AMD Instinct GPUs).
+#
+# This image installs the upstream `monai` package.
+#
+#   docker build -f Dockerfile.rocm -t monai:rocm .
+#
+#   docker run --device=/dev/kfd --device=/dev/dri --group-add video \
+#              --ipc=host --shm-size=8g -it monai:rocm
+#
+# Select the GPU architecture with '--build-arg AMDGPU_TARGETS=gfx942|gfx950'. The default is gfx942.
+
+ARG BASE_IMAGE=ubuntu:24.04
+FROM ${BASE_IMAGE}
+
+LABEL maintainer="monai.contact@gmail.com"
+
+ARG AMDGPU_TARGETS="gfx942"
+ARG ROCM_SERIES="10.0"
+ARG ROCM_INDEX_URL="https://stable.repo.amd.com/rocm/whl-next/"
+# Must match the python3 in BASE_IMAGE: it determines the venv's site-packages path below.
+ARG PYTHON_VERSION=3.12
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# ninja-build: required for MONAI's JIT C++/HIP extensions (torch.utils.cpp_extension).
+# libstdc++-13-dev + libopenslide-dev: HIP compile headers and whole-slide-image backend.
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+       ca-certificates curl git openssh-client \
+       python3-venv python3-pip python3-dev \
+       build-essential cmake ninja-build yasm \
+       libgomp1 libstdc++-13-dev \
+       libopenslide-dev libwebp-dev libzstd-dev \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
+RUN pip install --no-cache-dir --upgrade pip wheel
+
+RUN pip install --no-cache-dir --index-url ${ROCM_INDEX_URL} \
+      "rocm[libraries,devel,device-${AMDGPU_TARGETS}]==${ROCM_SERIES}.*" \
+      "torch[device-${AMDGPU_TARGETS}]" \
+      "torchvision[device-${AMDGPU_TARGETS}]" \
+      torchaudio \
+  && rocm-sdk init
+
+ENV ROCM_PATH="/opt/venv/lib/python${PYTHON_VERSION}/site-packages/_rocm_sdk_core"
+ENV ROCM_HOME="${ROCM_PATH}"
+ENV ROCM_DEVEL_PATH="/opt/venv/lib/python${PYTHON_VERSION}/site-packages/_rocm_sdk_devel"
+ENV ROCM_LIBRARIES_PATH="/opt/venv/lib/python${PYTHON_VERSION}/site-packages/_rocm_sdk_libraries"
+ENV PATH="${ROCM_PATH}/bin:${PATH}"
+ENV LD_LIBRARY_PATH="${ROCM_PATH}/lib:${ROCM_PATH}/lib/rocm_sysdeps/lib:${ROCM_PATH}/lib/llvm/lib:${ROCM_LIBRARIES_PATH}/lib"
+ENV CPATH="${ROCM_DEVEL_PATH}/include:/usr/lib/gcc/x86_64-linux-gnu/13/include"
+ENV LIBRARY_PATH="${ROCM_DEVEL_PATH}/lib:${ROCM_PATH}/lib"
+ENV AMDGPU_TARGETS=${AMDGPU_TARGETS}
+ENV PYTORCH_ROCM_ARCH=${AMDGPU_TARGETS}
+
+# hipcc expects bitcode at ROCM_PATH/amdgcn/bitcode; the rocm-sdk wheel places it one level deeper.
+RUN mkdir -p "${ROCM_PATH}/amdgcn" \
+  && ln -sf "${ROCM_PATH}/lib/llvm/amdgcn/bitcode" "${ROCM_PATH}/amdgcn/bitcode"
+
+# Prevent OpenBLAS from spawning one thread per core under MONAI's multiprocessing dataloaders.
+ENV OMP_NUM_THREADS=1
+
+WORKDIR /opt/monai
+
+COPY LICENSE CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md README.md versioneer.py setup.py pyproject.toml runtests.sh MANIFEST.in ./
+COPY tests ./tests
+COPY monai ./monai
+
+# Use print_dependencies.py rather than -e .[all,testing] to filter CUDA-only packages:
+#   cucim-cu* pulls in cuda-toolkit (~1.2 GB); nvidia-ml-py fails at import on ROCm; nni depends on it.
+# BUILD_MONAI=1 builds the C++/HIP extensions ahead of time; FORCE_CUDA=1 is required because the
+# build host has no GPU, so setup.py's `torch.cuda.is_available()` check would otherwise skip them.
+# Compilation itself needs only the toolkit and PYTORCH_ROCM_ARCH (set above), not a device.
+# pytest is not in the "testing" extra; it is added explicitly for running tests by hand.
+RUN python monai/config/print_dependencies.py build-system \
+      | xargs -d '\n' pip install --no-cache-dir --no-build-isolation \
+  && python monai/config/print_dependencies.py all testing \
+      | grep -vE '^cucim-cu|^nvidia-ml-py|^nni' > /tmp/rocm-requirements-$$.txt \
+  && BUILD_MONAI=1 FORCE_CUDA=1 pip install --no-cache-dir --no-build-isolation \
+      -r /tmp/rocm-requirements-$$.txt pytest -e . \
+  && rm -f /tmp/rocm-requirements-$$.txt
+
+# Required at runtime too: monai.config.deviceconfig gates USE_COMPILED on this variable, so
+# without it the extensions built above would be present but never used.
+ENV BUILD_MONAI=1
+
+# Set HIPCIM_INDEX_URL="" to build without WSI/cucim support.
+# CuImage is imported (not just cucim) because cucim uses lazy_loader -- a bare import
+# succeeds even when the native library is unresolvable. Failing here is deliberate: if
+# hipCIM was requested, an image where the cucim backends silently do not work is worse
+# than no image at all.
+ARG HIPCIM_INDEX_URL="https://pypi.amd.com/rocm-${ROCM_SERIES}.0/simple/"
+RUN if [ -n "${HIPCIM_INDEX_URL}" ]; then \
+      pip install --no-cache-dir --extra-index-url "${HIPCIM_INDEX_URL}" "amd-hipcim" \
+      && python -c "from cucim import CuImage"; \
+    else \
+      echo "hipCIM not installed; whole-slide-image (cucim) backends are unavailable."; \
+    fi
+
+RUN python - <<'PY'
+import torch, monai
+print('MONAI  :', monai.__version__)
+print('PyTorch:', torch.__version__)
+print('ROCm   :', torch.version.hip)
+try:
+    import cucim
+    from cucim import CuImage  # noqa: F401
+    print('hipCIM :', cucim.__version__)
+except ImportError as exc:
+    print('hipCIM : not available (%s)' % exc)
+PY
+
+CMD ["bash"]

monai/_extensions/loader.py

@@ -75,7 +75,9 @@ def load_module(
     source = glob(path.join(module_dir, "**", "*.cpp"), recursive=True)
     if torch.cuda.is_available():
         source += glob(path.join(module_dir, "**", "*.cu"), recursive=True)
-        platform_str += f"_{torch.version.cuda}"
+        # `torch.version.cuda` is None on a ROCm build, which would make every ROCm
+        # toolkit version share a single cache entry. Key on whichever is populated.
+        platform_str += f"_{torch.version.cuda or f'hip{torch.version.hip}'}"
 
     # Constructing compilation argument list.
     define_args = [] if not defines else [f"-D {key}={defines[key]}" for key in defines]

monai/apps/auto3dseg/bundle_gen.py

@@ -491,7 +491,7 @@ class BundleGen(AlgoGen):
         mlflow_tracking_uri: a tracking URI for MLflow server which could be local directory or address of
             the remote tracking Server; MLflow runs will be recorded locally in algorithms' model folder if
             the value is None.
-        mlfow_experiment_name: a string to specify the experiment name for MLflow server.
+        mlflow_experiment_name: a string to specify the experiment name for MLflow server.
     .. code-block:: bash
 
         python -m monai.apps.auto3dseg BundleGen generate --data_stats_filename="../algorithms/datastats.yaml"

monai/apps/detection/metrics/matching.py

@@ -193,7 +193,7 @@ def _matching_no_gt(
 
     Args:
         iou_thresholds: defined which IoU thresholds should be evaluated
-        dt_scores: predicted scores
+        pred_scores: predicted scores
         max_detections: maximum number of allowed detections per image.
             This functions uses this parameter to stay consistent with
             the actual matching function which needs this limit.

monai/data/dataset_summary.py

@@ -159,13 +159,20 @@ def calculate_statistics(self, foreground_threshold: int = 0):
             label, *_ = convert_data_type(data=label, output_type=torch.Tensor)
 
             image_foreground = image[torch.where(label > foreground_threshold)]
+            if image_foreground.numel() == 0:
+                continue
 
             voxel_max.append(image_foreground.max().item())
             voxel_min.append(image_foreground.min().item())
             voxel_ct += len(image_foreground)
             voxel_sum += image_foreground.sum()
             voxel_square_sum += torch.square(image_foreground).sum()
 
+        if voxel_ct == 0:
+            raise ValueError(
+                f"No foreground voxels found in any sample with {foreground_threshold=}; "
+                "set foreground_threshold=-1 to compute statistics over whole images."
+            )
         self.data_max, self.data_min = max(voxel_max), min(voxel_min)
         self.data_mean = (voxel_sum / voxel_ct).item()
         self.data_std = (torch.sqrt(voxel_square_sum / voxel_ct - self.data_mean**2)).item()
@@ -204,11 +211,17 @@ def calculate_percentiles(
             label, *_ = convert_data_type(data=label, output_type=torch.Tensor)
 
             intensities = image[torch.where(label > foreground_threshold)].tolist()
-            if sampling_flag:
-                intensities = intensities[::interval]
-            all_intensities.append(intensities)
+            if intensities:
+                if sampling_flag:
+                    intensities = intensities[::interval]
+                all_intensities.append(intensities)
 
         all_intensities = list(chain(*all_intensities))
+        if not all_intensities:
+            raise ValueError(
+                f"No foreground voxels found in any sample with {foreground_threshold=}; "
+                "set foreground_threshold=-1 to compute statistics over whole images."
+            )
         self.data_min_percentile, self.data_max_percentile = np.percentile(
             all_intensities, [min_percentile, max_percentile]
         )

monai/data/image_writer.py

@@ -266,12 +266,13 @@ def resample_if_needed(
                 The output data type of this method is always ``np.float32``.
         """
         orig_type = type(data_array)
-        data_array = convert_to_tensor(data_array, track_meta=True)
+        # Add the channel dimension before setting the affine so spatial_ndim is inferred correctly.
+        data_array = convert_to_tensor(data_array, track_meta=True)[None]
         if affine is not None:
             data_array.affine = convert_to_tensor(affine, track_meta=False)  # type: ignore
         resampler = SpatialResample(mode=mode, padding_mode=padding_mode, align_corners=align_corners, dtype=dtype)
         output_array = resampler(
-            data_array[None], dst_affine=target_affine, spatial_size=output_spatial_shape  # type: ignore
+            data_array, dst_affine=target_affine, spatial_size=output_spatial_shape  # type: ignore
         )
         # convert back at the end
         if isinstance(output_array, MetaTensor):

monai/data/utils.py

@@ -686,28 +686,40 @@ def worker_init_fn(worker_id: int) -> None:
     set_rnd(worker_info.dataset, seed=worker_info.seed)  # type: ignore[union-attr]
 
 
-def set_rnd(obj, seed: int) -> int:
+def set_rnd(obj, seed: int, _seen: set[int] | None = None) -> int:
     """
     Set seed or random state for all randomizable properties of obj.
 
     Args:
         obj: object to set seed or random state for.
         seed: set the random state with an integer seed.
+        _seen: internal set of already-visited object ids, used to guard against
+            infinite recursion on cyclic object graphs (e.g. OmegaConf/Hydra
+            configs whose child nodes back-reference their parent, see issue #8087).
     """
+    if _seen is None:
+        _seen = set()
     if isinstance(obj, (tuple, list)):  # ZipDataset.data is a list
-        _seed = seed
+        if id(obj) in _seen:
+            return seed
+        _seen.add(id(obj))
+        has_randomizable = False
         for item in obj:
-            _seed = set_rnd(item, seed=seed)
-        return seed if _seed == seed else seed + 1  # return a different seed if there are randomizable items
+            item_seed = set_rnd(item, seed=seed, _seen=_seen)
+            has_randomizable = has_randomizable or item_seed != seed
+        return seed + 1 if has_randomizable else seed
     if not hasattr(obj, "__dict__"):
         return seed  # no attribute
+    if id(obj) in _seen:
+        return seed  # already visited: avoid infinite recursion on cyclic references
+    _seen.add(id(obj))
     if hasattr(obj, "set_random_state"):
         obj.set_random_state(seed=seed % MAX_SEED)
         return seed + 1  # a different seed for the next component
     for key in obj.__dict__:
         if key.startswith("__"):  # skip the private methods
             continue
-        seed = set_rnd(obj.__dict__[key], seed=seed)
+        seed = set_rnd(obj.__dict__[key], seed=seed, _seen=_seen)
     return seed