From cd8b274f895cf8d2df10663caab7d5adc56f0437 Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Thu, 24 Sep 2026 22:31:49 +0200 Subject: [PATCH] Pin the performance floor gate's contract with tests check-perf-floors.sh is the only check between an order-of-magnitude regression and main, and it has no tests: scripts/ carries a test file for validate_georelays.py and two for the workflows around it, and nothing for the script that decides whether a slow hot path merges. Its behaviour is a shell retry loop around a Python heredoc that parses captured test output and picks exit codes -- exactly the kind of logic that drifts quietly. This runs the real script against fixture PERF logs and pins the documented contract: - every floor met passes; - the best value across appended attempts is what counts, so one retry that clears the floor is enough; - a below-floor result is re-measured and passes once it clears; - a below-floor result on every attempt fails (exit 1); - a floored benchmark that stops reporting fails without a retry (exit 3) -- a re-measurement that happened to produce the line would paper over a silently dropped benchmark; - no PERF lines and BITCHAT_SKIP_PERF_BASELINES=1 both skip (exit 0). The re-measure command is replaced by a stand-in that records each invocation and appends whatever the test says to the log the gate points it at, so the retry path runs in milliseconds with no toolchain, and a test that retries unexpectedly fails fast instead of building the app. One test pins a behaviour rather than endorsing it: a benchmark that reports a PERF line with no floor prints "consider adding a floor" and passes. That is the gate's one advisory branch -- removing a floored benchmark fails, adding an unfloored one does not -- and it is left as found so the follow-up flipping it is a three-line script change with a one-assertion test diff. The tests run in the app matrix entry immediately before the gate they describe. They also fall under the scripts/tests discovery in validate_georelays.yml, which is harmless. --- .github/workflows/swift-tests.yml | 6 + scripts/tests/test_check_perf_floors.py | 173 ++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 scripts/tests/test_check_perf_floors.py diff --git a/.github/workflows/swift-tests.yml b/.github/workflows/swift-tests.yml index e64683de..b287dfe1 100644 --- a/.github/workflows/swift-tests.yml +++ b/.github/workflows/swift-tests.yml @@ -121,6 +121,12 @@ jobs: BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log run: swift test --quiet --filter PerformanceBaselineTests + # The gate below is the only check between a benchmark regression and + # main; its exit-code contract is pinned by scripts/tests before it runs. + - name: Test the performance floor gate + if: matrix.name == 'app' + run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v + # Order-of-magnitude performance regression gate. Floors are deliberately # generous (see bitchatTests/Performance/perf-floors.json) so this # catches algorithmic regressions, never runner variance. If a metric diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py new file mode 100644 index 00000000..156d7dba --- /dev/null +++ b/scripts/tests/test_check_perf_floors.py @@ -0,0 +1,173 @@ +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh" + + +class CheckPerfFloorsTests(unittest.TestCase): + """The floor gate is what stands between an order-of-magnitude regression + and main, and it decides by parsing captured test output. These run the + real script against fixture logs and pin its exit-code contract: which + outcomes skip, which are re-measured, and which fail without a retry. + """ + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.invocations = self.tmp / "remeasure-invocations" + + def floors(self, **floors: int) -> Path: + path = self.tmp / "floors.json" + entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items()) + path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8") + return path + + def log(self, *lines: str) -> Path: + path = self.tmp / "perf-output.log" + path.write_text("".join(line + "\n" for line in lines), encoding="utf-8") + return path + + def remeasure(self, append: str | None = None) -> Path: + """A stand-in for `swift test`: records each invocation and appends one + line to the log the gate points it at, the way a real re-run would.""" + script = self.tmp / "remeasure.sh" + body = ["#!/bin/sh", f'echo run >> "{self.invocations}"'] + if append is not None: + body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"") + script.write_text("\n".join(body) + "\n", encoding="utf-8") + script.chmod(0o755) + return script + + def run_gate( + self, + log: Path, + floors: Path, + *, + remeasure: Path | None = None, + attempts: int | None = None, + skip: bool = False, + ) -> subprocess.CompletedProcess: + env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")} + # Never fall through to the default `swift test` re-measure: a test + # that unexpectedly retries must fail fast, not build the app. + env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure()) + if attempts is not None: + env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts) + if skip: + env["BITCHAT_SKIP_PERF_BASELINES"] = "1" + return subprocess.run( + [str(GATE), str(log), str(floors)], + cwd=self.tmp, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def remeasure_count(self) -> int: + if not self.invocations.exists(): + return 0 + return len(self.invocations.read_text(encoding="utf-8").splitlines()) + + def test_every_floor_met_passes(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all benchmarks at or above their floors", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_best_value_across_appended_attempts_is_what_counts(self) -> None: + # Re-measurements append to the same log, so one line clearing the + # floor is enough even when an earlier attempt sat under it. + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec", "PERF[a.one]: 500 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + + def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("re-measuring (attempt 2", result.stdout) + + def test_below_floor_on_every_attempt_fails(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"), + attempts=2, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout) + self.assertIn("treating as a real regression", result.stderr) + + def test_missing_floored_benchmark_fails_without_a_retry(self) -> None: + # A re-measurement that happened to produce the missing line would + # paper over a silently dropped benchmark, so this must not retry. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"), + ) + + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout) + + def test_unfloored_benchmark_is_reported_but_does_not_fail(self) -> None: + # Pinned as found, not endorsed: the gate fails when a floored + # benchmark stops reporting, but a benchmark that reports with no floor + # only prints advice. That asymmetry is the subject of the follow-up. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("NO-FLOOR c.new: 1000 ops/sec (consider adding a floor)", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_output_without_perf_lines_skips(self) -> None: + # Package-only matrix entries produce no benchmarks; the gate must + # not fail them for it. + result = self.run_gate( + self.log("Test Suite 'All tests' passed", "Executed 12 tests"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("no PERF lines", result.stdout) + + def test_skip_flag_bypasses_the_gate_entirely(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1 ops/sec"), + self.floors(**{"a.one": 100}), + skip=True, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + +if __name__ == "__main__": + unittest.main()