bench: fit the anchor tree to vscode's node_modules, 4 KiB files

tests/bench/charts.py

@@ -180,7 +180,8 @@ def _speedup_by_fixture(theme: Theme, run: Run, summary: Summary):
     default = 4 if 4 in run.plan.threads else max(run.plan.threads)
     shown = [p for p in summary.pairs if p.tool.threads == default or not _swept(run, p.tool.tool)]
     pairs = [p for p in shown if p.tool.cache == "warm"]
-    fixtures = sorted({p.tool.fixture for p in pairs}, key=lambda f: int(f[1:]))
+    present = {p.tool.fixture for p in pairs}
+    fixtures = [f.id for f in run.plan.fixtures if f.id in present]  # the registry's order
     tools = sorted({p.tool.tool for p in pairs}, key=lambda t: (t != HIGHLIGHT, not _swept(run, t), t))
     rest = iter(theme.rest)
     width = min(0.8 / len(tools), 0.17)

tests/bench/registry.py

@@ -77,17 +77,31 @@ def _linux_tools(remmy: Path) -> tuple[Tool, ...]:
     )
 
 
+# Each id names the tree's shape and file count, and is how results are labelled everywhere: pytest ids, chart
+# files, and Bencher (demo/balanced-38k/warm/remmy@4). test_fixture_ids_match_their_trees keeps the counts honest.
+#
+# Every file is 4 KiB unless a tree says otherwise. In real node_modules trees ~80% of files are <= 4 KiB (median
+# ~1 KiB), and APFS and ext4 allocate them one 4 KiB block each, so every file frees a block like a real one does.
+SMALL = 4 << 10
 FIXTURES = (
-    Fixture(id="F0", title="Anchor", depth=3, fanout=8, files=100),
-    Fixture(id="F1", title="Shallow and wide", depth=1, fanout=512, files=100),
-    Fixture(id="F2", title="Deep and narrow", depth=9, fanout=2, files=57),
-    Fixture(id="F3", title="Small files (4 KiB)", depth=3, fanout=8, files=100, size=4096),
-    Fixture(id="F4", title="Large files (1 MiB)", depth=2, fanout=6, files=45, size=1 << 20),
-    Fixture(id="F5", title="Tiny tree", depth=1, fanout=4, files=25),
-    Fixture(id="F6", title="Huge tree", depth=3, fanout=10, files=300),
-    # F0's file count in one directory: a single scan task, no subdirs to spread.
-    Fixture(id="F7", title="Flat directory", depth=0, fanout=0, files=58_500),
+    # The anchor every other tree varies one thing of: vscode's node_modules shape, made uniform. vscode c17dab9
+    # (`npm ci --ignore-scripts`, macOS arm64, 2026-09-28) has 5,538 dirs and 39,966 files: 7.2 files per dir, 2.7
+    # subdirs per non-leaf dir, 63% leaf dirs, files a median 4 levels down. This is 5,461 dirs x 7 files.
+    Fixture(id="balanced-38k", title="Anchor", depth=6, fanout=4, files=7, size=SMALL),
+    # The anchor's dirs and files, all under one directory: many small scan tasks at one level.
+    Fixture(id="wide-38k", title="Shallow and wide", depth=1, fanout=5460, files=7, size=SMALL),
+    # An 11-level binary tree, 4,095 dirs x 9 files: long dependency chains.
+    Fixture(id="deep-36k", title="Deep and narrow", depth=11, fanout=2, files=9, size=SMALL),
+    # 43 dirs x 45 files of 1 MiB (1.9 GiB): unlink cost dominated by freeing extents.
+    Fixture(id="large-1mib-files", title="Large files (1 MiB)", depth=2, fanout=6, files=45, size=1 << 20),
+    # 5 dirs x 25 files: startup cost.
+    Fixture(id="tiny-125", title="Tiny tree", depth=1, fanout=4, files=25, size=SMALL),
+    # The anchor one level deeper, 21,845 dirs x 7 files: a monorepo's node_modules (bitwarden/clients has 124k files).
+    Fixture(id="balanced-152k", title="Huge tree", depth=7, fanout=4, files=7, size=SMALL),
+    # The anchor's file count in one directory: a single scan task, no subdirs to spread.
+    Fixture(id="flat-38k", title="Flat directory", depth=0, fanout=0, files=38_227, size=SMALL),
 )
+ANCHOR = FIXTURES[0].id
 
 
 def plan(name: str, remmy: Path, platform: str = sys.platform) -> Plan:
@@ -98,7 +112,7 @@ def plan(name: str, remmy: Path, platform: str = sys.platform) -> Plan:
         return Plan(
             name="demo",
             fixtures=FIXTURES[:1],
-            caches={"F0": (Cache.WARM,)},
+            caches={ANCHOR: (Cache.WARM,)},
             tools=tuple(t for t in tools if t.name in ("rm", "xargs", "remmy")),
             threads=(1, 4),
             sampling={Cache.WARM: Sampling(reps=6, rounds=2, warmup=1)},
@@ -109,7 +123,7 @@ def plan(name: str, remmy: Path, platform: str = sys.platform) -> Plan:
         return Plan(
             name="full",
             fixtures=FIXTURES,
-            caches={f.id: (Cache.WARM, Cache.COLD) if f.id == "F0" else (Cache.WARM,) for f in FIXTURES},
+            caches={f.id: (Cache.WARM, Cache.COLD) if f.id == ANCHOR else (Cache.WARM,) for f in FIXTURES},
             tools=tools,
             threads=(1, 2, 4, 8),
             sampling={

tests/bench/test_stats.py

@@ -4,6 +4,7 @@
 import json
 import math
 import random
+import re
 from pathlib import Path
 
 import numpy as np
@@ -105,7 +106,7 @@ def test_synthetic_run_recovers_planted_effects(demo_plan):
     summary = stats.summarize(schema.Run(plan=demo_plan, samples=tuple(samples)))
     verdicts = {p.tool.label: p.verdict for p in summary.pairs}
     assert verdicts["remmy@4"] is Verdict.DISTINCT
-    assert summary.pair(CellKey("F0", "remmy", 4, Cache.WARM)).time_ratio < 1
+    assert summary.pair(CellKey("balanced-38k", "remmy", 4, Cache.WARM)).time_ratio < 1
     assert verdicts["remmy@1"] is Verdict.TIE
     assert verdicts["xargs@1"] is Verdict.DISTINCT
 
@@ -151,7 +152,7 @@ def test_interval_rejects_reversed_bounds():
 
 
 def test_cell_key_labels():
-    assert str(CellKey("F0", "remmy", 4, Cache.WARM)) == "F0/warm/remmy@4"
+    assert str(CellKey("balanced-38k", "remmy", 4, Cache.WARM)) == "balanced-38k/warm/remmy@4"
 
 
 def test_bmf_carries_the_summary_bencher_tracks(demo_plan, tmp_path):
@@ -165,15 +166,28 @@ def test_bmf_carries_the_summary_bencher_tracks(demo_plan, tmp_path):
     summary = session.summary()
     doc = bmf.convert(json.loads(session.write_json(summary).read_text()))
 
-    assert set(doc) == {f"demo/F0/warm/{c.label}" for c in cells}
-    remmy = summary.pair(CellKey("F0", "remmy", 4, Cache.WARM))
-    ratio = doc["demo/F0/warm/remmy@4"]["time-ratio"]
+    assert set(doc) == {f"demo/balanced-38k/warm/{c.label}" for c in cells}
+    remmy = summary.pair(CellKey("balanced-38k", "remmy", 4, Cache.WARM))
+    ratio = doc["demo/balanced-38k/warm/remmy@4"]["time-ratio"]
     assert ratio == {"value": remmy.time_ratio, "lower_value": remmy.time_ratio_ci95.lo,
                      "upper_value": remmy.time_ratio_ci95.hi}  # fmt: skip
     # Only the subject is judged on a ratio; the reference and competitors are controls.
     assert all("time-ratio" not in m for name, m in doc.items() if "/remmy@" not in name)
-    latency = doc["demo/F0/warm/rm@1"]["latency"]
+    latency = doc["demo/balanced-38k/warm/rm@1"]["latency"]
     assert latency["lower_value"] <= latency["value"] <= latency["upper_value"]
     assert 0.5e9 < latency["value"] < 2e9  # ns
-    rate = doc["demo/F0/warm/rm@1"]["throughput"]
+    rate = doc["demo/balanced-38k/warm/rm@1"]["throughput"]
     assert rate["lower_value"] <= rate["value"] <= rate["upper_value"]
+
+
+def _count(n: int) -> str:
+    return f"{n // 1000}k" if n >= 1000 else str(n)
+
+
+@pytest.mark.parametrize("fixture", registry.FIXTURES, ids=lambda f: f.id)
+def test_fixture_ids_match_their_trees(fixture):
+    """An id is a label people read: its file count must be the tree's, and it must be safe in a path or URL."""
+    assert re.fullmatch(r"[a-z0-9]+(-[a-z0-9]+)*", fixture.id)
+    files = fixture.expected.files
+    counts = re.findall(r"(?<![a-z])(\d+k?)(?=-|$)", fixture.id)
+    assert all(c == _count(files) for c in counts if not c.endswith(("kib", "mib"))), (fixture.id, files)