Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ai/data/feature_extractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"motion2",
)

DEFAULT_VMAF_BINARY = Path("libvmaf") / "build-cpu" / "tools" / "vmaf"
DEFAULT_VMAF_BINARY = Path("core") / "build-cpu" / "tools" / "vmaf"

# Full feature set the fork's extractors can produce (per Research-0026,
# extended by ADR-0559 to include SpEED chroma/temporal features).
Expand Down
56 changes: 49 additions & 7 deletions ai/lpips_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,15 @@ def _parity_check(onnx_path: Path, atol: float = 1e-4) -> None:
print(f"[ok] parity: max|Δ|={diff:.3e} ≤ atol={atol:.1e}")


def _write_sidecar(onnx_path: Path, opset: int) -> Path:
sidecar = onnx_path.with_suffix(".json")
payload = {
def _write_sidecar(
onnx_path: Path,
opset: int,
*,
sidecar_path: Path | None = None,
run_provenance: dict | None = None,
) -> Path:
sidecar = sidecar_path if sidecar_path is not None else onnx_path.with_suffix(".json")
payload: dict = {
"input_name": "ref",
"kind": "fr",
"name": "vmaf_tiny_lpips_sq_v1",
Expand All @@ -195,6 +201,8 @@ def _write_sidecar(onnx_path: Path, opset: int) -> Path:
"onnx_opset": opset,
"output_name": "score",
}
if run_provenance is not None:
payload["run_provenance"] = run_provenance
sidecar.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
return sidecar

Expand All @@ -208,11 +216,18 @@ def _sha256(path: Path) -> str:


def main(argv: list[str] | None = None) -> int:
repo_root = Path(__file__).resolve().parent.parent
script_path = Path(__file__).resolve()
repo_root = script_path.parent.parent
default_out = repo_root / "model" / "tiny" / "lpips_sq.onnx"

parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", type=Path, default=default_out)
parser.add_argument("--output", "--out", dest="output", type=Path, default=default_out)
parser.add_argument(
"--sidecar",
type=Path,
default=None,
help="Path for the JSON sidecar (default: <output>.json).",
)
parser.add_argument("--opset", type=int, default=17)
parser.add_argument("--skip-parity", action="store_true")
args = parser.parse_args(argv)
Expand All @@ -225,13 +240,40 @@ def main(argv: list[str] | None = None) -> int:
"sidecar + registry should use the emitted value",
file=sys.stderr,
)
_write_sidecar(args.output, effective_opset)
# Build run_provenance lazily so aiutils import errors surface only when
# the sidecar is actually written (keeps the module importable without aiutils).
run_provenance: dict | None = None
try:
_ai_src = str(script_path.parent / "src")
if _ai_src not in sys.path:
sys.path.insert(0, _ai_src)
from aiutils.run_manifest import build_run_provenance

run_provenance = build_run_provenance(
entrypoint=script_path,
repo_root=repo_root,
argv=sys.argv[1:] if argv is None else list(argv),
args=vars(args),
outputs={"onnx": args.output},
)
except ImportError:
pass
_write_sidecar(
args.output,
effective_opset,
sidecar_path=args.sidecar,
run_provenance=run_provenance,
)
if not args.skip_parity:
_parity_check(args.output)

sha = _sha256(args.output)
size = args.output.stat().st_size
print(f"[ok] wrote {args.output.relative_to(repo_root)} ({size} bytes)")
try:
display_path = args.output.relative_to(repo_root)
except ValueError:
display_path = args.output
print(f"[ok] wrote {display_path} ({size} bytes)")
print(f" sha256 {sha}")
print(" add this digest to model/tiny/registry.json")
return 0
Expand Down
4 changes: 2 additions & 2 deletions ai/scripts/bvi_dvc_to_full_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -693,8 +693,8 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument(
"--vmaf-bin",
type=Path,
default=REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf",
help="Path to the libvmaf CLI binary.",
default=REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf",
help="Path to the vmaf CLI binary.",
)
ap.add_argument(
"--model",
Expand Down
6 changes: 3 additions & 3 deletions ai/scripts/extract_k150k_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -1112,7 +1112,7 @@ def main() -> int:
ap.add_argument(
"--vmaf-bin",
type=Path,
default=REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf",
default=REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf",
help=(
"Path to the fork vmaf binary (built with ssimulacra2 + motion_v2). "
"Default: core/build-cpu/tools/vmaf. Passing a CUDA-capable binary "
Expand All @@ -1123,7 +1123,7 @@ def main() -> int:
ap.add_argument(
"--cpu-vmaf-bin",
type=Path,
default=REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf",
default=REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf",
help=(
"CPU vmaf binary used for residual CPU-only feature passes when "
"--vmaf-bin points at a CUDA-capable binary. Default: "
Expand Down Expand Up @@ -1234,7 +1234,7 @@ def main() -> int:
print(
f"error: vmaf binary not found: {args.vmaf_bin}\n"
"Build the fork vmaf binary with:\n"
" meson setup core/build-cpu libvmaf -Denable_cuda=false "
" meson setup core/build-cpu core -Denable_cuda=false "
"--buildtype=release && ninja -C core/build-cpu\n"
"Then re-run with --vmaf-bin core/build-cpu/tools/vmaf",
file=sys.stderr,
Expand Down
4 changes: 2 additions & 2 deletions ai/scripts/konvid_to_vmaf_pairs.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,8 @@ def main(argv: list[str] | None = None) -> int:
ap.add_argument(
"--vmaf-bin",
type=Path,
default=REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf",
help="Path to the libvmaf CLI binary.",
default=REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf",
help="Path to the vmaf CLI binary.",
)
ap.add_argument(
"--model",
Expand Down
5 changes: 0 additions & 5 deletions ai/src/aiutils/jsonl_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
from __future__ import annotations

import json
<<<<<<< HEAD
import math
from pathlib import Path
from typing import Any, Iterator
Expand Down Expand Up @@ -44,10 +43,6 @@ def dumps_jsonl_row(obj: dict, **kwargs: Any) -> str:
sanitised = _sanitize_nonfinite(obj)
kwargs.setdefault("sort_keys", True)
return json.dumps(sanitised, **kwargs) + "\n"
=======
from pathlib import Path
from typing import Iterator
>>>>>>> 24bb5daf89 (docs: post-merge-train sweep — VMAFx + core/ path refs, ADR index, state.md)


def iter_jsonl(path: Path) -> Iterator[tuple[int, dict]]:
Expand Down
2 changes: 1 addition & 1 deletion ai/src/vmaf_train/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@

app = typer.Typer(add_completion=False, no_args_is_help=True)
console = Console()
DEFAULT_VMAF_BINARY = Path("libvmaf") / "build-cpu" / "tools" / "vmaf"
DEFAULT_VMAF_BINARY = Path("core") / "build-cpu" / "tools" / "vmaf"


def _write_cli_report_json(
Expand Down
2 changes: 1 addition & 1 deletion ai/src/vmaf_train/data/feature_dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def _normalize_pixfmt(fmt: str) -> str:
"motion2",
)

DEFAULT_VMAF_BINARY = Path("libvmaf") / "build-cpu" / "tools" / "vmaf"
DEFAULT_VMAF_BINARY = Path("core") / "build-cpu" / "tools" / "vmaf"

# vmaf's CLI takes feature-extractor names (``adm``, ``vif``, ``motion``) while
# the JSON output labels individual metrics (``adm2``, ``vif_scale0``, …).
Expand Down
6 changes: 0 additions & 6 deletions ai/src/vmaf_train/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,7 @@

import hashlib
import json
<<<<<<< HEAD
import math
=======
>>>>>>> 24bb5daf89 (docs: post-merge-train sweep — VMAFx + core/ path refs, ADR index, state.md)
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Any
Expand Down Expand Up @@ -117,7 +114,6 @@ def register(
def load(sidecar_path: Path) -> ModelMetadata:
doc: dict[str, Any] = json.loads(sidecar_path.read_text())
return ModelMetadata(**doc)
<<<<<<< HEAD


def _sanitize_nonfinite(obj: Any) -> Any:
Expand Down Expand Up @@ -168,5 +164,3 @@ def write_registry_json(path: Path, payload: dict, **kwargs: Any) -> None:
**kwargs: Forwarded to :func:`dumps_registry_json` / :func:`json.dumps`.
"""
path.write_text(dumps_registry_json(payload, **kwargs) + "\n", encoding="utf-8")
=======
>>>>>>> 24bb5daf89 (docs: post-merge-train sweep — VMAFx + core/ path refs, ADR index, state.md)
2 changes: 1 addition & 1 deletion ai/tests/test_chug_extract_features_smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def _resolve_vmaf_binary() -> Path | None:
return Path(env)
for candidate in (
_REPO_ROOT / "build" / "tools" / "vmaf",
_REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf",
_REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf",
):
if candidate.is_file():
return candidate
Expand Down
2 changes: 1 addition & 1 deletion ai/tests/test_e2e_frame_to_score.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
from vmaf_train.data.feature_dump import DEFAULT_FEATURES, Entry, dump_features # noqa: E402

REPO_ROOT = Path(__file__).resolve().parents[2]
VMAF_BIN = REPO_ROOT / "libvmaf" / "build-cpu" / "tools" / "vmaf"
VMAF_BIN = REPO_ROOT / "core" / "build-cpu" / "tools" / "vmaf"
YUV_DIR = REPO_ROOT / "python" / "test" / "resource" / "yuv"
REF_YUV = YUV_DIR / "src01_hrc00_576x324.yuv"
DIS_YUV = YUV_DIR / "src01_hrc01_576x324.yuv"
Expand Down
4 changes: 2 additions & 2 deletions ai/tests/test_feature_extractor_defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def test_default_vmaf_binary_uses_fork_cpu_build(monkeypatch) -> None:
monkeypatch.delenv("VMAF_BIN", raising=False)

assert feature_extractor.default_vmaf_binary() == (
Path("libvmaf") / "build-cpu" / "tools" / "vmaf"
Path("core") / "build-cpu" / "tools" / "vmaf"
)


Expand All @@ -25,4 +25,4 @@ def test_default_vmaf_binary_respects_env(monkeypatch) -> None:


def test_vmaf_train_feature_dump_default_uses_fork_cpu_build() -> None:
assert (Path("libvmaf") / "build-cpu" / "tools" / "vmaf") == feature_dump.DEFAULT_VMAF_BINARY
assert (Path("core") / "build-cpu" / "tools" / "vmaf") == feature_dump.DEFAULT_VMAF_BINARY
2 changes: 1 addition & 1 deletion ai/tests/test_train_konvid_mos_head.py
Original file line number Diff line number Diff line change
Expand Up @@ -711,7 +711,7 @@ def test_smoke_run_produces_allowlist_conformant_onnx(tmp_path: Path) -> None:
model = onnx_pkg.load(str(onnx_path))
onnx_pkg.checker.check_model(model)
ops = {n.op_type for n in model.graph.node}
allowlist_text = (REPO_ROOT / "libvmaf" / "src" / "dnn" / "op_allowlist.c").read_text()
allowlist_text = (REPO_ROOT / "core" / "src" / "dnn" / "op_allowlist.c").read_text()
for op in ops:
assert f'"{op}"' in allowlist_text, f"op {op} missing from op_allowlist.c"

Expand Down
10 changes: 10 additions & 0 deletions changelog.d/fixed/ai-post-rename.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
### Fixed

- **ai/ post-ADR-0700 path cleanup**: replaced all stale `libvmaf/build-cpu/tools/vmaf`
and `libvmaf/src/dnn/` default paths with `core/build-cpu/tools/vmaf` and `core/src/dnn/`
across `ai/data/`, `ai/src/vmaf_train/`, `ai/scripts/`, and `ai/tests/`. The `VMAF_BIN`
env-var override is unaffected. Fixed a wrong `meson setup core/build-cpu libvmaf` hint
(source-dir argument was the old directory name; corrected to `core`). Resolved leftover
git conflict markers in `ai/src/aiutils/jsonl_utils.py` and `ai/src/vmaf_train/registry.py`
that were causing import-time `SyntaxError`. Added missing `--sidecar` / `--out` flags
and `run_provenance` emission to `ai/lpips_export.py` (fixes `test_lpips_export` failure).
Loading