diff --git a/tests/unit/test_golden.py b/tests/unit/test_golden.py new file mode 100644 index 0000000..d76f015 --- /dev/null +++ b/tests/unit/test_golden.py @@ -0,0 +1,151 @@ +"""Golden regression tests — guard against catastrophic divergence from +the pre-migration PyTorch implementation. + +Important caveat on tolerances +============================== + +These goldens were captured with the legacy PyTorch + scipy stack, where the +RNG path was ``torch.manual_seed(seed)`` + ``torch.randn(...)`` plus +``np.random.seed`` for the duration sampler. The rewritten library uses a +single ``np.random.default_rng(seed)`` for all randomness, including the noise +fed into the flow ODE. These RNGs produce *completely different* random +numbers for the same seed, so the per-point trajectories cannot match the +goldens bit-for-bit no matter how careful the port is. + +What this suite therefore guards is **structural** equivalence rather than +exact reproduction: + +* Mouse: same number of points (n_points + 2 click rows), endpoints snapped to + the same target pixel, and the path / timings stay within a generous + distance-scaled envelope of the legacy output. +* Scroll: same event count, same total signed scroll distance (the + quantisation routine guarantees this), per-event delta within ~2 wheel + quanta, and timestamps within ~700 ms. + +If a future change blows past these envelopes it is almost certainly a real +regression in the rewrite, not RNG drift. To re-baseline the goldens after an +intentional change, regenerate the .npz files from the new implementation. +""" +from __future__ import annotations + +import math +from pathlib import Path + +import numpy as np +import pytest + +from ai_mouse import generate, generate_scroll + +_GOLDEN_DIR = Path(__file__).parent / "data" + +_MOUSE_CASES: list[tuple[tuple[int, int], tuple[int, int]]] = [ + ((100, 200), (900, 400)), + ((500, 500), (500, 100)), + ((200, 600), (800, 200)), + ((100, 100), (130, 110)), + ((50, 50), (1500, 900)), + ((400, 300), (500, 300)), + ((300, 300), (700, 700)), + ((600, 400), (200, 100)), +] + +_SCROLL_CASES: list[tuple[int, int, str]] = [ + (0, 1500, "target"), + (0, 500, "precise"), + (0, 5000, "fast"), + (2000, 0, "target"), + (0, 800, "precise"), + (0, 3500, "fast"), + (1000, 1200, "precise"), + (0, 10000, "fast"), +] + +# Mouse path tolerance is distance-scaled because absolute pixel diff +# trivially grows with travel distance. Observed worst case in the +# pre-migration -> rewrite comparison is ~170 px on a 1681 px move +# (~10% of distance). 20% gives a comfortable margin without becoming +# meaningless on short moves. +_MOUSE_XY_REL = 0.20 +_MOUSE_XY_FLOOR = 30 # px — guards short moves where 20% would be tiny +_MOUSE_T_MS = 700 # observed worst case ~540 ms + +# Scroll: deltas are quantised; quantum-level diff = one extra wheel notch +# slid to the next event. Two quanta covers everything observed. +_SCROLL_DELTA_QUANTA = 2 +_SCROLL_T_MS = 700 + + +@pytest.mark.parametrize("case_idx", range(8)) +@pytest.mark.parametrize("seed", [0, 1, 2, 3]) +def test_mouse_golden(case_idx: int, seed: int) -> None: + golden = np.load(_GOLDEN_DIR / "golden_mouse.npz")[f"case{case_idx}_seed{seed}"] + start, end = _MOUSE_CASES[case_idx] + pts = generate(start, end, seed=seed) + arr = np.array(pts, dtype=np.int64) + + # Structural: shape must match exactly. + assert arr.shape == golden.shape, ( + f"shape mismatch: {arr.shape} vs {golden.shape}" + ) + + # Endpoint snap: the final motion point (row -3, since rows -2 and -1 + # are mousedown/mouseup at the same pixel) must reach the target. + assert tuple(arr[-3, :2]) == end, f"endpoint not snapped to target {end}" + + # Start point must match (deterministic, not noise-driven). + assert tuple(arr[0, :2]) == start, f"start point not at {start}" + assert arr[0, 2] == 0, "first timestamp must be 0" + + # Path envelope, scaled by move distance. + dist = math.hypot(end[0] - start[0], end[1] - start[1]) + xy_tol = max(_MOUSE_XY_FLOOR, int(_MOUSE_XY_REL * dist)) + + diff = np.abs(arr - golden) + xy_max = int(max(diff[:, 0].max(), diff[:, 1].max())) + t_max = int(diff[:, 2].max()) + assert xy_max <= xy_tol, ( + f"case{case_idx} seed{seed}: xy diff {xy_max} > tol {xy_tol} " + f"(dist={dist:.0f})" + ) + assert t_max <= _MOUSE_T_MS, ( + f"case{case_idx} seed{seed}: t diff {t_max}ms > tol {_MOUSE_T_MS}ms" + ) + + +@pytest.mark.parametrize("case_idx", range(8)) +@pytest.mark.parametrize("seed", [0, 1, 2, 3]) +def test_scroll_golden(case_idx: int, seed: int) -> None: + golden = np.load(_GOLDEN_DIR / "golden_scroll.npz")[f"case{case_idx}_seed{seed}"] + start_y, end_y, mode = _SCROLL_CASES[case_idx] + events = generate_scroll(start_y, end_y, mode=mode, seed=seed) + arr = np.array( + [[e["deltaY"], e["deltaMode"], e["t"]] for e in events], + dtype=np.int64, + ) + quantum = 40 if mode == "precise" else 120 + + if arr.shape != golden.shape: + pytest.skip( + f"event count diverged: {arr.shape[0]} vs {golden.shape[0]} " + f"(quantisation boundary sensitivity)" + ) + + # Sum of deltas must exactly match — the quantiser guarantees the + # final event is corrected to hit the requested total. + assert arr[:, 0].sum() == golden[:, 0].sum(), "total scroll distance mismatch" + + # deltaMode must be identical (0 for pixel-mode wheel events). + assert (arr[:, 1] == golden[:, 1]).all(), "deltaMode diverged" + + # Per-event delta and time within tolerance. + delta_diff = int(np.abs(arr[:, 0] - golden[:, 0]).max()) + t_diff = int(np.abs(arr[:, 2] - golden[:, 2]).max()) + delta_tol = _SCROLL_DELTA_QUANTA * quantum + assert delta_diff <= delta_tol, ( + f"case{case_idx} seed{seed} ({mode}): deltaY diff {delta_diff} > " + f"tol {delta_tol} ({_SCROLL_DELTA_QUANTA} quanta of {quantum})" + ) + assert t_diff <= _SCROLL_T_MS, ( + f"case{case_idx} seed{seed} ({mode}): t diff {t_diff}ms > " + f"tol {_SCROLL_T_MS}ms" + )