Add exhaustive/extensive tests

.github/workflows/main.yml

@@ -14,7 +14,7 @@ env:
 jobs:
   test:
     name: Build and test
-    timeout-minutes: 20
+    timeout-minutes: 25
     strategy:
       fail-fast: false
       matrix:
@@ -186,13 +186,64 @@ jobs:
         rustup component add rustfmt
     - run: cargo fmt -- --check
 
+  calculate_matrix:
+    name: Calculate job matrix
+    runs-on: ubuntu-24.04
+    outputs:
+      matrix: ${{ steps.script.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 100
+      - name: Fetch pull request ref
+        run: git fetch origin "$GITHUB_REF:$GITHUB_REF"
+      - run: python3 ci/calculate-exhaustive-matrix.py >> "$GITHUB_OUTPUT"
+        id: script
+
+  extensive:
+    name: Extensive tests for ${{ matrix.ty }}
+    needs:
+      - test
+      - calculate_matrix
+    runs-on: ubuntu-24.04
+    timeout-minutes: 80
+    strategy:
+      matrix:
+        # Use the output from `calculate_matrix` to calculate the matrix
+        include: ${{ fromJSON(needs.calculate_matrix.outputs.matrix).matrix }}
+    env:
+      CHANGED: ${{ matrix.changed }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Rust
+        run: |
+          rustup update nightly --no-self-update
+          rustup default nightly
+
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Download musl source
+        run: ./ci/download-musl.sh
+
+      - name: Run extensive tests
+        run: |
+          LIBM_EXTENSIVE_TESTS="$CHANGED" cargo t \
+            --features test-multiprecision,build-musl,unstable \
+            --release -- extensive
+
+      - name: Print test logs if available
+        run: if [ -f "target/test-log.txt" ]; then cat target/test-log.txt; fi
+        shell: bash
+
   success:
     needs:
       - test
       - builtins
       - benchmarks
       - msrv
       - rustfmt
+      - extensive
     runs-on: ubuntu-24.04
     # GitHub branch protection is exceedingly silly and treats "jobs skipped because a dependency
     # failed" as success. So we have to do some contortions to ensure the job fails if any of its

ci/calculate-exhaustive-matrix.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Calculate which exhaustive tests should be run as part of CI.
+
+This dynamically prepares a list of TODO
+"""
+
+import subprocess as sp
+import sys
+import json
+from dataclasses import dataclass
+from os import getenv
+from pathlib import Path
+from typing import TypedDict
+
+
+REPO_ROOT = Path(__file__).parent.parent
+GIT = ["git", "-C", REPO_ROOT]
+
+# Don't run exhaustive tests if these files change, even if they contaiin a function
+# definition.
+IGNORE_FILES = [
+    "src/math/support/",
+    "src/libm_helper.rs",
+    "src/math/arch/intrinsics.rs",
+]
+
+TYPES = ["f16", "f32", "f64", "f128"]
+
+
+class FunctionDef(TypedDict):
+    """Type for an entry in `function-definitions.json`"""
+
+    sources: list[str]
+    type: str
+
+
+@dataclass
+class Context:
+    gh_ref: str | None
+    changed: list[Path]
+    defs: dict[str, FunctionDef]
+
+    def __init__(self) -> None:
+        self.gh_ref = getenv("GITHUB_REF")
+        self.changed = []
+        self._init_change_list()
+
+        with open(REPO_ROOT.joinpath("etc/function-definitions.json")) as f:
+            defs = json.load(f)
+
+        defs.pop("__comment", None)
+        self.defs = defs
+
+    def _init_change_list(self):
+        """Create a list of files that have been changed. This uses GITHUB_REF if
+        available, otherwise a diff between `HEAD` and `master`.
+        """
+
+        # For pull requests, GitHub creates a ref `refs/pull/1234/merge` (1234 being
+        # the PR number), and sets this as `GITHUB_REF`.
+        ref = self.gh_ref
+        eprint(f"using ref `{ref}`")
+        if ref is None or "merge" not in ref:
+            # If the ref is not for `merge` then we are not in PR CI
+            eprint("No diff available for ref")
+            return
+
+        # The ref is for a dummy merge commit. We can extract the merge base by
+        # inspecting all parents (`^@`).
+        merge_sha = sp.check_output(
+            GIT + ["show-ref", "--hash", ref], text=True
+        ).strip()
+        merge_log = sp.check_output(GIT + ["log", "-1", merge_sha], text=True)
+        eprint(f"Merge:\n{merge_log}\n")
+
+        parents = (
+            sp.check_output(GIT + ["rev-parse", f"{merge_sha}^@"], text=True)
+            .strip()
+            .splitlines()
+        )
+        assert len(parents) == 2, f"expected two-parent merge but got:\n{parents}"
+        base = parents[0].strip()
+        incoming = parents[1].strip()
+
+        eprint(f"base: {base}, incoming: {incoming}")
+        textlist = sp.check_output(
+            GIT + ["diff", base, incoming, "--name-only"], text=True
+        )
+        self.changed = [Path(p) for p in textlist.splitlines()]
+
+    @staticmethod
+    def _ignore_file(fname: str) -> bool:
+        return any(fname.startswith(pfx) for pfx in IGNORE_FILES)
+
+    def changed_routines(self) -> dict[str, list[str]]:
+        """Create a list of routines for which one or more files have been updated,
+        separated by type.
+        """
+        routines = set()
+        for name, meta in self.defs.items():
+            # Don't update if changes to the file should be ignored
+            sources = (f for f in meta["sources"] if not self._ignore_file(f))
+
+            # Select changed files
+            changed = [f for f in sources if Path(f) in self.changed]
+
+            if len(changed) > 0:
+                eprint(f"changed files for {name}: {changed}")
+                routines.add(name)
+
+        ret = {}
+        for r in sorted(routines):
+            ret.setdefault(self.defs[r]["type"], []).append(r)
+
+        return ret
+
+    def make_workflow_output(self) -> str:
+        changed = self.changed_routines()
+        ret = []
+        for ty in TYPES:
+            ty_changed = changed.get(ty, [])
+            item = {
+                "ty": ty,
+                "changed": ",".join(ty_changed),
+            }
+            ret.append(item)
+        output = json.dumps({"matrix": ret}, separators=(",", ":"))
+        eprint(f"output: {output}")
+        return output
+
+
+def eprint(*args, **kwargs):
+    """Print to stderr."""
+    print(*args, file=sys.stderr, **kwargs)
+
+
+def main():
+    ctx = Context()
+    output = ctx.make_workflow_output()
+    print(f"matrix={output}")
+
+
+if __name__ == "__main__":
+    main()

crates/libm-test/Cargo.toml

@@ -26,12 +26,14 @@ short-benchmarks = []
 [dependencies]
 anyhow = "1.0.90"
 az = { version = "1.2.1", optional = true }
+indicatif = { version = "0.17.9", default-features = false }
 libm = { path = "../..", features = ["unstable-test-support"] }
 libm-macros = { path = "../libm-macros" }
 musl-math-sys = { path = "../musl-math-sys", optional = true }
 paste = "1.0.15"
 rand = "0.8.5"
 rand_chacha = "0.3.1"
+rayon = "1.10.0"
 rug = { version = "1.26.1", optional = true, default-features = false, features = ["float", "std"] }
 
 [target.'cfg(target_family = "wasm")'.dependencies]
@@ -43,11 +45,18 @@ rand = { version = "0.8.5", optional = true }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", default-features = false, features = ["cargo_bench_support"] }
+libtest-mimic = "0.8.1"
 
 [[bench]]
 name = "random"
 harness = false
 
+[[test]]
+# No harness so that we can skip tests at runtime based on env. Prefixed with
+# `z` so these tests get run last.
+name = "z_extensive"
+harness = false
+
 [lints.rust]
 # Values from the chared config.rs used by `libm` but not the test crate
 unexpected_cfgs = { level = "warn", check-cfg = [

crates/libm-test/src/gen.rs

@@ -2,6 +2,7 @@
 
 pub mod domain_logspace;
 pub mod edge_cases;
+pub mod extensive;
 pub mod random;
 
 /// A wrapper to turn any iterator into an `ExactSizeIterator`. Asserts the final result to ensure

crates/libm-test/src/gen/extensive.rs

@@ -0,0 +1,133 @@
+use std::fmt;
+use std::ops::RangeInclusive;
+
+use libm::support::MinInt;
+
+use crate::domain::HasDomain;
+use crate::gen::KnownSize;
+use crate::op::OpITy;
+use crate::run_cfg::{int_range, iteration_count};
+use crate::{CheckCtx, GeneratorKind, MathOp, logspace};
+
+/// Generate a sequence of inputs that either cover the domain in completeness (for smaller float
+/// types and single argument functions) or provide evenly spaced inputs across the domain with
+/// approximately `u32::MAX` total iterations.
+pub trait ExtensiveInput<Op> {
+    fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> + Send;
+}
+
+/// Construct an iterator from `logspace` and also calculate the total number of steps expected
+/// for that iterator.
+fn logspace_steps<Op>(
+    start: Op::FTy,
+    end: Op::FTy,
+    ctx: &CheckCtx,
+    argnum: usize,
+) -> (impl Iterator<Item = Op::FTy> + Clone, u64)
+where
+    Op: MathOp,
+    OpITy<Op>: TryFrom<u64, Error: fmt::Debug>,
+    RangeInclusive<OpITy<Op>>: Iterator,
+{
+    let max_steps = iteration_count(ctx, GeneratorKind::Extensive, argnum);
+    let max_steps = OpITy::<Op>::try_from(max_steps).unwrap_or(OpITy::<Op>::MAX);
+    let iter = logspace(start, end, max_steps);
+
+    // `logspace` can't implement `ExactSizeIterator` because of the range, but its size hint
+    // should be accurate (assuming <= usize::MAX iterations).
+    let size_hint = iter.size_hint();
+    assert_eq!(size_hint.0, size_hint.1.unwrap());
+
+    (iter, size_hint.0.try_into().unwrap())
+}
+
+macro_rules! impl_extensive_input {
+    ($fty:ty) => {
+        impl<Op> ExtensiveInput<Op> for ($fty,)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+            Op: HasDomain<Op::FTy>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = Op::DOMAIN.range_start();
+                let end = Op::DOMAIN.range_end();
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let iter0 = iter0.map(|v| (v,));
+                KnownSize::new(iter0, steps0)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for ($fty, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+                let iter =
+                    iter0.flat_map(move |first| iter1.clone().map(move |second| (first, second)));
+                let count = steps0.checked_mul(steps1).unwrap();
+                KnownSize::new(iter, count)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for ($fty, $fty, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+
+                let (iter0, steps0) = logspace_steps::<Op>(start, end, ctx, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+                let (iter2, steps2) = logspace_steps::<Op>(start, end, ctx, 2);
+
+                let iter = iter0
+                    .flat_map(move |first| iter1.clone().map(move |second| (first, second)))
+                    .flat_map(move |(first, second)| {
+                        iter2.clone().map(move |third| (first, second, third))
+                    });
+                let count = steps0.checked_mul(steps1).unwrap().checked_mul(steps2).unwrap();
+
+                KnownSize::new(iter, count)
+            }
+        }
+
+        impl<Op> ExtensiveInput<Op> for (i32, $fty)
+        where
+            Op: MathOp<RustArgs = Self, FTy = $fty>,
+        {
+            fn get_cases(ctx: &CheckCtx) -> impl ExactSizeIterator<Item = Self> {
+                let start = <$fty>::NEG_INFINITY;
+                let end = <$fty>::INFINITY;
+
+                let iter0 = int_range(ctx, GeneratorKind::Extensive, 0);
+                let steps0 = iteration_count(ctx, GeneratorKind::Extensive, 0);
+                let (iter1, steps1) = logspace_steps::<Op>(start, end, ctx, 1);
+
+                let iter =
+                    iter0.flat_map(move |first| iter1.clone().map(move |second| (first, second)));
+                let count = steps0.checked_mul(steps1).unwrap();
+
+                KnownSize::new(iter, count)
+            }
+        }
+    };
+}
+
+impl_extensive_input!(f32);
+impl_extensive_input!(f64);
+
+/// Create a test case iterator for extensive inputs.
+pub fn get_test_cases<Op>(
+    ctx: &CheckCtx,
+) -> impl ExactSizeIterator<Item = Op::RustArgs> + Send + use<'_, Op>
+where
+    Op: MathOp,
+    Op::RustArgs: ExtensiveInput<Op>,
+{
+    Op::RustArgs::get_cases(ctx)
+}