build(meson): stop icx and the clang-cl drivers warning about the project's own compile flags

.github/test-suites.json

@@ -51,6 +51,16 @@
       "fail_on_skip": "vmaf binary|vmaf CLI|golden YUV|Netflix .*(YUV|fixtures)|checkerboard YUVs",
       "runs": "pytest ai/tests/ ai/sidecar/tests/ with ai/requirements-dev-lock.txt"
     },
+    {
+      "name": "mini-retrain",
+      "paths": ["ai/e2e/"],
+      "checks": ["Tiny AI"],
+      "source_paths": ["ai/", "model/tiny/registry.schema.json", "testdata/ref_576x324_48f.yuv", "testdata/dis_576x324_48f.yuv"],
+      "install": {"python": "3.14", "locks": ["requirements/locks/package-build.txt", "ai/requirements-dev-lock.txt"], "editable": ["ai"]},
+      "pytest": {"timeout": 900, "method": "signal"},
+      "fail_on_skip": ".*",
+      "runs": "pytest ai/e2e/ (the mini retrain end to end) in the Tiny AI job (which builds the vmaf CLI the test runs) and nightly in mini-retrain.yml (ADR-1898)"
+    },
     {
       "name": "mcp",
       "paths": ["mcp-server/vmaf-mcp/tests/"],

.github/workflows/mini-retrain.yml

@@ -0,0 +1,76 @@
+name: mini-retrain
+
+# The retrain tooling end to end on a generated corpus (ADR-1898, issue #1246).
+# Runs nightly and on demand; pull requests that touch ai/ run the same tests in
+# the required Tiny AI job. A red run means a stage of the one-shot retrain would have stopped: extraction,
+# feature checks, combination, training, export, validation, registry or gate.
+
+on:
+  schedule:
+    - cron: "17 5 * * *"  # 05:17 UTC, after the nightly bisect
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: mini-retrain-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mini-retrain:
+    name: Mini Retrain nightly
+    runs-on: ubuntu-26.04
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Install build deps
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ninja-build nasm pkg-config python3-pip python3-venv
+          sudo pip3 install --break-system-packages --quiet --require-hashes \
+            -r requirements/locks/build.txt
+
+      - name: Build the vmaf CLI (CPU, no DNN)
+        run: |
+          meson setup build core -Denable_cuda=false -Denable_sycl=false --buildtype=release
+          ninja -C build tools/vmaf
+
+      - name: Install the training stack (hash-locked)
+        env:
+          TMPDIR: ${{ runner.temp }}
+        run: |
+          venv="${RUNNER_TEMP}/mini-retrain-venv"
+          python3 -m venv "${venv}"
+          "${venv}/bin/pip" install --require-hashes -r requirements/locks/package-build.txt
+          "${venv}/bin/pip" install --require-hashes -r ai/requirements-dev-lock.txt
+          "${venv}/bin/pip" install --no-deps --no-build-isolation -e ai
+
+      - name: Unit tests of the stage runner and the checks
+        run: |
+          "${RUNNER_TEMP}/mini-retrain-venv/bin/pytest" \
+            ai/tests/test_pipeline.py ai/tests/test_retrain_checks.py \
+            ai/tests/test_mini_corpus.py -q --timeout=120
+
+      - name: Mini retrain end to end
+        env:
+          VMAF_BIN: ${{ github.workspace }}/build/tools/vmaf
+          VMAFX_CONTAINER_IMAGE_ID: github-hosted-ubuntu-26.04
+        run: |
+          "${RUNNER_TEMP}/mini-retrain-venv/bin/pytest" ai/e2e -v -rs \
+            --timeout=900 --timeout-method=signal \
+            --basetemp="${RUNNER_TEMP}/mini-retrain-runs"
+
+      - name: Keep the stage manifests and the gate report
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: mini-retrain-manifests
+          path: |
+            ${{ runner.temp }}/mini-retrain-runs/**/stages/*.json
+            ${{ runner.temp }}/mini-retrain-runs/**/reports/*.json
+          if-no-files-found: warn
+          retention-days: 14

.github/workflows/tests-and-quality-gates.yml

@@ -368,7 +368,7 @@ jobs:
     # required-aggregator
     name: Tiny AI
     runs-on: ubuntu-26.04
-    timeout-minutes: 25
+    timeout-minutes: 35
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
         with:
@@ -484,6 +484,17 @@ jobs:
           VMAF_BIN_FOR_TESTS: ${{ github.workspace }}/core/build/tools/vmaf
         run: |
           "${RUNNER_TEMP}/vmaf-train-venv/bin/pytest" ai/tests/ ai/sidecar/tests/ -v -rs --timeout=60 --timeout-method=signal
+      # Suite mini-retrain (ADR-1898): the retrain tooling end to end on a generated
+      # corpus, with the vmaf CLI this job built. Any skip fails the step.
+      - name: Run the mini retrain end to end
+        if: steps.impact.outputs.tiny_ai == 'true'
+        env:
+          VMAF_BIN: ${{ github.workspace }}/core/build/tools/vmaf
+          VMAFX_CONTAINER_IMAGE_ID: github-hosted-ubuntu-26.04
+        run: |
+          "${RUNNER_TEMP}/vmaf-train-venv/bin/pytest" ai/e2e/ -v -rs \
+            --timeout=900 --timeout-method=signal \
+            --basetemp="${RUNNER_TEMP}/mini-retrain-runs"
       # ADR-0174 / T5-3b: enforce that every quantised model in the
       # registry stays inside its per-model PLCC drop budget. Skips
       # entries whose `quant_mode == "fp32"` (nothing to gate).

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "1.0.0-rc.4"
+  ".": "1.0.0-rc.3"
 }

CHANGELOG.md

@@ -6,6 +6,15 @@
 ## [Unreleased]
 ### Added
 
+- **Mini retrain and a resumable stage runner for the retrain tooling** (ADR-1898, issue #1246).
+  `make mini-retrain` runs extraction, feature checks, combination, training and export of
+  `vmaf_tiny_v2` to `v4` and `fr_regressor_v1`, validation, registry validation and a PLCC / SROCC / RMSE
+  gate on a generated 144-row corpus in about 40 seconds. Every stage writes a manifest with seed,
+  digests, library versions, lock digest, container id and resource use; a killed run resumes from the
+  manifests; a missing or corrupt input stops the run with the stage name before anything runs. The
+  Tiny AI job runs it for changes under `ai/`, and a nightly workflow runs it too. See the runbook section 13.
+
+
 - A weekly research radar over public video-quality sources: a public source registry (`docs/research/radar/sources.yaml`), a scheduled digest workflow (`research-radar.yml`, `scripts/research/radar_collect.py`) and a documented triage procedure with a licence and patent gate ([ADR-2171](docs/adr/2171-research-radar.md), [docs/research/radar/](docs/research/radar/README.md)).
 
 
@@ -41,6 +50,40 @@
   changed.
 
 
+- **icx and the clang-cl style drivers stop warning about our own compile flags.** `icx` and `icpx`
+  reported `-ffp-contract=off` after `-fp-model=precise` as `-Woverriding-option` on every compile
+  (7,300 times in one CI leg). The strict policy now spells `-fp-model=precise -fno-fast-math
+  -fcomplex-arithmetic=full -ffp-contract=off`; on 182 translation units of this tree the objects
+  are byte-identical to the old spelling ([ADR-2170](docs/adr/2170-warnings-are-errors-per-leg.md)).
+  clang-cl and icx-cl are no longer offered `-pedantic`, `-fvisibility=hidden` and
+  `-fvisibility-inlines-hidden`, which they ignored with a warning per compile.
+
+
+- **Option tables, extractor tables and tag declarations no longer print compiler warnings.**
+  The clang, gcc, icpx and Apple clang legs reported `-Wmissing-field-initializers`
+  (`{NULL}` / `{0}` option terminators, positional test tables), `-Wreorder-init-list` and
+  `-Wc99-designator` (the Metal option and extractor tables), `-Wmismatched-tags`
+  (`VmafThreadLocaleState`, declared `struct` in the C header and `class` in the C++ file),
+  `-Wimplicit-fallthrough`, `-Wkeyword-macro` (vendored cJSON redefining `true` / `false` in
+  C23), `-Wtautological-constant-out-of-range-compare` (`vmaf_next_fex_capacity()` on 64-bit
+  hosts) and `-Wmacro-redefined` (`DIV_ROUND_UP` in the HIP ADM twin). Every fix is
+  value-preserving: initialisers are reordered or completed, `[[fallthrough]]` replaces
+  comments, the capacity check compares in `size_t`. No score, option default or exported
+  symbol changes.
+
+
+- **Unused code, ignored attributes and deprecated calls no longer print compiler warnings.**
+  Test tables and helpers that only a skipped or disabled configuration reaches are compiled only
+  there (`-Wunused-function` / `-Wunused-variable` in the ms_ssim_decimate, cambi, ssimulacra2 and
+  read_pictures tests, the registry helpers of `model_loader.c` on Windows, the HIP error mappers
+  without `hipcc`); `VMAF_EXPORT` is empty for GCC on MinGW, where the visibility attribute was
+  ignored and drew a warning under LTO; the Win32 pthread shim spells `__stdcall` only in the host
+  pass of a SYCL build; `VmafRef` is constructed with count 1 instead of calling the deprecated
+  `std::atomic_init()`; the ONNX Runtime headers are `-isystem`; and a test that linked with an
+  explicit `link_language : 'cpp'` no longer repeats `-lc++` on macOS. No score, symbol or
+  option changes.
+
+
 ### Fixed
 
 - **`-qpfile` works on libx264, and the saliency tools no longer run a libx264
@@ -61,6 +104,14 @@
   encode (two tests failed on that, depending on the order they ran in).
 
 
+- **`motion` was NaN in every row of every extracted feature table.** libvmaf emits no
+  `integer_motion` key; the first-order motion score is `VMAF_integer_feature_motion_sad_score`.
+  `ai/data/feature_extractor.py` now reads it, so the `motion` column of `FULL_FEATURES` holds values.
+  Tables extracted before this change carry an all-NaN `motion` column and the `verify_features` stage of
+  the mini retrain refuses them. `extract_full_features.py` also gains `--assume-dims WxH` for corpora
+  that are not 1920x1080.
+
+
 - The last MSVC warnings of the first Windows run after the zero-warning series
   are fixed: the CUDA / HIP decouple helpers and the x86 motion round constant
   shift in 64 bits (C4334; the operand never exceeds 30 bits), and three
@@ -365,6 +416,15 @@ They are recorded in full, unedited, in
   `docs/backends/metal/index.md`, `docs/development/cross-backend-gate.md`.
 
 
+- **Mini retrain and a resumable stage runner for the retrain tooling** (ADR-1898, issue #1246).
+  `make mini-retrain` runs extraction, feature checks, combination, training and export of
+  `vmaf_tiny_v2` to `v4` and `fr_regressor_v1`, validation, registry validation and a PLCC / SROCC / RMSE
+  gate on a generated 144-row corpus in about 40 seconds. Every stage writes a manifest with seed,
+  digests, library versions, lock digest, container id and resource use; a killed run resumes from the
+  manifests; a missing or corrupt input stops the run with the stage name before anything runs. The
+  Tiny AI job runs it for changes under `ai/`, and a nightly workflow runs it too. See the runbook section 13.
+
+
 - **`motion_five_frame_window` works, and the four `vmaf_v1.0.16_hfr_*` models
   score.** The option of the `motion` and `motion_v2` extractors takes each
   frame's SAD against the frame two back, and `motion2` from the SADs of the
@@ -3113,6 +3173,40 @@ make `core/AGENTS.md` a generated index over `AGENTS.d/` topic pages ([ADR-1454]
   explicitly where the higher-quality encode is wanted. See ADR-1562.
 
 
+- **icx and the clang-cl style drivers stop warning about our own compile flags.** `icx` and `icpx`
+  reported `-ffp-contract=off` after `-fp-model=precise` as `-Woverriding-option` on every compile
+  (7,300 times in one CI leg). The strict policy now spells `-fp-model=precise -fno-fast-math
+  -fcomplex-arithmetic=full -ffp-contract=off`; on 182 translation units of this tree the objects
+  are byte-identical to the old spelling ([ADR-2170](docs/adr/2170-warnings-are-errors-per-leg.md)).
+  clang-cl and icx-cl are no longer offered `-pedantic`, `-fvisibility=hidden` and
+  `-fvisibility-inlines-hidden`, which they ignored with a warning per compile.
+
+
+- **Option tables, extractor tables and tag declarations no longer print compiler warnings.**
+  The clang, gcc, icpx and Apple clang legs reported `-Wmissing-field-initializers`
+  (`{NULL}` / `{0}` option terminators, positional test tables), `-Wreorder-init-list` and
+  `-Wc99-designator` (the Metal option and extractor tables), `-Wmismatched-tags`
+  (`VmafThreadLocaleState`, declared `struct` in the C header and `class` in the C++ file),
+  `-Wimplicit-fallthrough`, `-Wkeyword-macro` (vendored cJSON redefining `true` / `false` in
+  C23), `-Wtautological-constant-out-of-range-compare` (`vmaf_next_fex_capacity()` on 64-bit
+  hosts) and `-Wmacro-redefined` (`DIV_ROUND_UP` in the HIP ADM twin). Every fix is
+  value-preserving: initialisers are reordered or completed, `[[fallthrough]]` replaces
+  comments, the capacity check compares in `size_t`. No score, option default or exported
+  symbol changes.
+
+
+- **Unused code, ignored attributes and deprecated calls no longer print compiler warnings.**
+  Test tables and helpers that only a skipped or disabled configuration reaches are compiled only
+  there (`-Wunused-function` / `-Wunused-variable` in the ms_ssim_decimate, cambi, ssimulacra2 and
+  read_pictures tests, the registry helpers of `model_loader.c` on Windows, the HIP error mappers
+  without `hipcc`); `VMAF_EXPORT` is empty for GCC on MinGW, where the visibility attribute was
+  ignored and drew a warning under LTO; the Win32 pthread shim spells `__stdcall` only in the host
+  pass of a SYCL build; `VmafRef` is constructed with count 1 instead of calling the deprecated
+  `std::atomic_init()`; the ONNX Runtime headers are `-isystem`; and a test that linked with an
+  explicit `link_language : 'cpp'` no longer repeats `-lc++` on macOS. No score, symbol or
+  option changes.
+
+
 - **zimg picture conversion allows approximate gamma (port of Netflix/vmaf `5c3f4fb90`).**
   `vmaf_picture_convert()` builds its zimg graph with `allow_approximate_gamma`, as FFmpeg's
   `zscale` does: exact transfer functions are about 20 times slower for PQ and change scores
@@ -4268,6 +4362,14 @@ The `vmaf` command-line tool now exits with the same status on every platform: a
   ([ADR-1558](docs/adr/1558-codec-block-encoding-from-sidecar.md)).
 
 
+- **`motion` was NaN in every row of every extracted feature table.** libvmaf emits no
+  `integer_motion` key; the first-order motion score is `VMAF_integer_feature_motion_sad_score`.
+  `ai/data/feature_extractor.py` now reads it, so the `motion` column of `FULL_FEATURES` holds values.
+  Tables extracted before this change carry an all-NaN `motion` column and the `verify_features` stage of
+  the mini retrain refuses them. `extract_full_features.py` also gains `--assume-dims WxH` for corpora
+  that are not 1920x1080.
+
+
 - **The Gitleaks check scans only the commit it checked out.** It ran
   `git log --all` over a full-history checkout, so a finding on any branch in
   the repository, including a commit a force-push had already replaced,

Makefile

@@ -121,6 +121,7 @@ $(NINJA): $(VENV_PIP)
 RUFF_VERSION  := 0.16.10
 BLACK_VERSION := 26.10.0
 
+.PHONY: mini-retrain mini-retrain-test
 .PHONY: lint-tools
 lint-tools: $(VENV_PIP)
 	$(VENV_PIP) install --quiet --require-hashes -r requirements/locks/dev-linters.txt
@@ -554,6 +555,17 @@ test-netflix-golden: build-golden
 	CUDA_VISIBLE_DEVICES="" VMAF_FORCE_BACKEND=cpu VMAF_BUILD_DIR="$(CURDIR)/$(GOLDEN_BUILD_DIR)" PYTHONPATH=$(CURDIR)/python python3 -m pytest \
 	    $(GOLDEN_PYTEST_ARGS)
 
+# The retrain tooling end to end on a generated corpus (docs/ai/retrain-runbook-1246.md, ADR-1898).
+# Needs a built vmaf CLI (VMAF_BIN or build/tools/vmaf) and the ai/ venv (torch, onnxruntime).
+#   make mini-retrain [MINI_RETRAIN_DIR=runs/mini] [VMAF_BIN=build/tools/vmaf]
+MINI_RETRAIN_DIR ?= runs/mini
+mini-retrain:
+	python3 ai/scripts/mini_retrain.py run --run-dir "$(MINI_RETRAIN_DIR)" \
+	    --vmaf-bin "$(or $(VMAF_BIN),build/tools/vmaf)"
+
+mini-retrain-test:
+	python3 -m pytest ai/e2e -v --timeout=900
+
 # The test suites a change affects, in cached hash-locked venvs (docs/development/test-suites.md).
 #   make test-affected BASE=origin/master HEAD=HEAD [VMAF_BIN=build/tools/vmaf]
 test-affected:

ai/AGENTS.d/mini-retrain.md

@@ -0,0 +1,14 @@
+---
+paths:
+  - ai/scripts/mini_retrain.py
+  - ai/src/aiutils/pipeline.py
+  - ai/src/aiutils/mini_corpus.py
+  - ai/src/aiutils/retrain_checks.py
+  - ai/e2e/test_mini_retrain_e2e.py
+  - .github/workflows/mini-retrain.yml
+invariant: The retrain tooling runs end to end in CI; each stage has a manifest; a killed run resumes; NaN fails a gate.
+---
+<!-- markdownlint-disable MD013 MD060 -->
+# Mini retrain and the stage runner
+
+- [ADR-1898](../../docs/adr/1898-mini-retrain-pipeline.md) — **A script the retrain runs is a stage of `mini_retrain.py` with the runbook's flags.** Adding a trainer, exporter or evaluator to `docs/ai/retrain-runbook-1246.md` means adding its stage (inputs, outputs, `stable` set, check) to `build_stages()` and a planted-defect case to `ai/e2e/test_mini_retrain_e2e.py` in the same PR. `aiutils.pipeline.run_pipeline()` resumes only on a `complete` manifest with an unchanged key and intact outputs; never write a stage that updates its input in place (use `Stage.copies`). `aiutils.retrain_checks.gate_verdict()` fails a NaN metric; do not replace it with `plcc < gate`, which passes NaN. `ai/e2e/` is not under the 60 s `ai/tests` limit and is run by the Tiny AI job and nightly.

ai/AGENTS.md

@@ -73,6 +73,7 @@ at runtime. Stack: PyTorch + Lightning → ONNX.
 | `scripts/extract_k150k_features.py`, `tests/test_extract_k150k_features.py` | [k150k-extraction](AGENTS.d/k150k-extraction.md) | Parquet written at-end only; staging file WAL; scratch on /dev/shm; fail-loud on empty frames; misuse guard. |
 | `scripts/analyze_knob_sweep.py`, `tests/test_knob_sweep_analysis.py` | [knob-sweep](AGENTS.d/knob-sweep.md) | Pareto frontiers per (source, codec, rc_mode); structural regressions (>=7 of 9 sources) must not ship. |
 | `scripts/konvid_to_full_features.py`, `scripts/konvid_to_vmaf_pairs.py`, `scripts/combine_full_feature_parquets.py` | [konvid-1k-corpus](AGENTS.d/konvid-1k-corpus.md) | KoNViD-1k refresh requires fresh fork binary; stable balanced hash fold assignment; replay manifest. |
+| `scripts/mini_retrain.py`, `src/aiutils/pipeline.py`, `src/aiutils/mini_corpus.py`, `src/aiutils/retrain_checks.py`, `e2e/test_mini_retrain_e2e.py`, `/.github/workflows/mini-retrain.yml` | [mini-retrain](AGENTS.d/mini-retrain.md) | The retrain tooling runs end to end in CI; each stage has a manifest; a killed run resumes; NaN fails a gate. |
 | `/docs/ai/training-data.md`, `/docs/ai/models/*.md`, `/model/tiny/registry.json`, `/scripts/ci/tests/test_model_card_dataset_terms.py` | [model-card-dataset-terms](AGENTS.d/model-card-dataset-terms.md) | Card of a model on outside data quotes the data's terms verbatim from training-data.md; a test holds it. |
 | `src/vmaf_train/registry.py`, `src/vmaf_train/train.py`, `src/vmaf_train/data/datasets.py` | [model-metadata-pydantic](AGENTS.d/model-metadata-pydantic.md) | Operator-facing TrainConfig, ModelMetadata, ManifestEntry use pydantic v2 BaseModels; internal stay @dataclass. |
 | `scripts/konvid_150k_to_corpus_jsonl.py`, `scripts/lsvq_to_corpus_jsonl.py`, `scripts/youtube_ugc_to_corpus_jsonl.py`, `scripts/waterloo_ivc_to_corpus_jsonl.py` | [mos-corpus-adapters](AGENTS.d/mos-corpus-adapters.md) | MOS corpus adapters share row schema; manifest-csv strict; cross-corpus rescaling is trainer-side concern. |

ai/data/feature_extractor.py

@@ -216,10 +216,20 @@ def _extractors_for(metrics: tuple[str, ...]) -> list[str]:
     return seen
 
 
+# libvmaf emits no ``integer_motion`` key: the first-order motion score of the
+# ``motion`` extractor is reported as ``VMAF_integer_feature_motion_sad_score``.
+# Without this alias the ``motion`` column of FULL_FEATURES was NaN in every row
+# of every extracted corpus (ADR-1898).
+_METRIC_KEY_ALIASES: dict[str, str] = {"motion": "VMAF_integer_feature_motion_sad_score"}
+
+
 def _lookup(metrics: dict, name: str):
     """libvmaf may emit ``integer_<name>`` for fixed-point kernels."""
     if name in metrics:
         return metrics[name]
+    alias = _METRIC_KEY_ALIASES.get(name)
+    if alias is not None and metrics.get(alias) is not None:
+        return metrics[alias]
     val = metrics.get(f"integer_{name}")
     if val is not None:
         return val