diff --git a/.github/workflows/swift-tests.yml b/.github/workflows/swift-tests.yml index e64683de..b287dfe1 100644 --- a/.github/workflows/swift-tests.yml +++ b/.github/workflows/swift-tests.yml @@ -121,6 +121,12 @@ jobs: BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log run: swift test --quiet --filter PerformanceBaselineTests + # The gate below is the only check between a benchmark regression and + # main; its exit-code contract is pinned by scripts/tests before it runs. + - name: Test the performance floor gate + if: matrix.name == 'app' + run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v + # Order-of-magnitude performance regression gate. Floors are deliberately # generous (see bitchatTests/Performance/perf-floors.json) so this # catches algorithmic regressions, never runner variance. If a metric diff --git a/bitchatTests/Performance/perf-floors.json b/bitchatTests/Performance/perf-floors.json index 3426d105..d28fa1c5 100644 --- a/bitchatTests/Performance/perf-floors.json +++ b/bitchatTests/Performance/perf-floors.json @@ -12,8 +12,9 @@ "magnitude regressions while never flaking on a slow runner.", "Raise floors deliberately after intentional performance improvements;", "lower them only with a written justification in the PR. If a benchmark is", - "renamed or removed, update this file in the same change - the gate fails", - "when a floored benchmark stops reporting.", + "added, renamed or removed, update this file in the same change - the gate", + "fails when a floored benchmark stops reporting or when a benchmark reports", + "with no floor.", "Checked by scripts/check-perf-floors.sh against captured swift-test output." ], "_units": "operations per second, matching each benchmark's PERF line", diff --git a/scripts/check-perf-floors.sh b/scripts/check-perf-floors.sh index b51eb390..8e567418 100755 --- a/scripts/check-perf-floors.sh +++ b/scripts/check-perf-floors.sh @@ -36,9 +36,12 @@ # Fails when: # - any benchmark reports throughput below its floor on every attempt # (exit 1), or -# - PERF lines are present but a floored benchmark is missing — a -# silently-dropped benchmark must be an explicit floors-file change and -# is not retried (exit 3). +# - the floors file is missing, unreadable or malformed — a setup error +# that no re-measurement can fix, so it is not retried (exit 2), or +# - PERF lines are present but a floored benchmark is missing, or a +# benchmark reports with no floor — the floors file and the benchmark +# suite must change together in both directions, and neither drift is +# retried (exit 3). set -euo pipefail @@ -80,10 +83,12 @@ case "$OUTPUT_FILE" in esac # Exit codes: 0 = all floors met, 1 = below floor (retryable — noise vs -# regression undecided), 3 = floored benchmark missing (not retryable). +# regression undecided), 2 = unreadable or malformed floors file, 3 = floors +# and benchmarks out of step (neither retryable). check_floors() { OUTPUT_FILE="$OUTPUT_FILE" FLOORS_FILE="$FLOORS_FILE" python3 - <<'PYEOF' import json +import math import os import re import sys @@ -91,8 +96,24 @@ import sys output_file = os.environ["OUTPUT_FILE"] floors_file = os.environ["FLOORS_FILE"] -with open(floors_file) as f: - floors = json.load(f)["floors"] +def malformed(detail): + # A setup error, not runner noise: exit 2 so the loop does not re-measure. + print(f"perf-floors: unreadable or malformed floors file '{floors_file}': {detail}", file=sys.stderr) + sys.exit(2) + +try: + with open(floors_file) as f: + document = json.load(f) +except (OSError, ValueError) as error: + malformed(error) +floors = document.get("floors") if isinstance(document, dict) else None +if not isinstance(floors, dict): + malformed('it needs a "floors" object') +for name, floor in floors.items(): + # NaN would fail every comparison, so a below-floor value would pass. + is_number = isinstance(floor, (int, float)) and not isinstance(floor, bool) + if not is_number or not math.isfinite(floor): + malformed(f'the floor for "{name}" is not a finite number') # PERF[delivery.storeUpdate]: 158862 updates/sec (avg 3.147 ms per pass of 500, 10 passes) pattern = re.compile(r"PERF\[([^\]]+)\]:\s*([0-9]+(?:\.[0-9]+)?)\s*(\S+)/sec") @@ -111,6 +132,7 @@ with open(output_file, errors="replace") as f: below_floor = [] missing = [] +unfloored = [] print(f"perf-floors: checking {len(measured)} benchmark(s) against {len(floors)} floor(s)") for name in sorted(set(floors) | set(measured)): floor = floors.get(name) @@ -121,7 +143,9 @@ for name in sorted(set(floors) | set(measured)): continue value, unit = measured[name] if floor is None: - print(f" NO-FLOOR {name}: {value:.0f} {unit}/sec (consider adding a floor)") + unfloored.append( + f" NO-FLOOR {name}: {value:.0f} {unit}/sec reports a PERF line but has " + f"no floor (add one to perf-floors.json in the same change)") continue status = "OK" if value >= floor else "BELOW" line = f" {status:8} {name}: {value:.0f} {unit}/sec (floor {floor})" @@ -131,9 +155,9 @@ for name in sorted(set(floors) | set(measured)): f" BELOW {name}: {value:.0f} {unit}/sec is under floor {floor} " f"({value / floor * 100:.0f}% of floor)") -if missing: - print("\nperf-floors: FAILED — floored benchmark(s) missing from the output:") - print("\n".join(missing + below_floor)) +if missing or unfloored: + print("\nperf-floors: FAILED — floors and benchmarks are out of step:") + print("\n".join(missing + unfloored + below_floor)) sys.exit(3) if below_floor: @@ -161,7 +185,8 @@ while true; do # Below floor: retry to separate runner noise from regression. ;; *) - # Missing benchmark or parse/setup error: re-measuring can't help. + # Floors and benchmarks out of step, or a setup error such as a + # malformed floors file: re-measuring can't help. exit "$gate_status" ;; esac diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py new file mode 100644 index 00000000..92bb07cd --- /dev/null +++ b/scripts/tests/test_check_perf_floors.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh" + + +class CheckPerfFloorsTests(unittest.TestCase): + """The floor gate is what stands between an order-of-magnitude regression + and main, and it decides by parsing captured test output. These run the + real script against fixture logs and pin its exit-code contract: which + outcomes skip, which are re-measured, and which fail without a retry. + """ + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.invocations = self.tmp / "remeasure-invocations" + + def floors(self, **floors: int) -> Path: + path = self.tmp / "floors.json" + entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items()) + path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8") + return path + + def log(self, *lines: str) -> Path: + path = self.tmp / "perf-output.log" + path.write_text("".join(line + "\n" for line in lines), encoding="utf-8") + return path + + def remeasure(self, append: str | None = None) -> Path: + """A stand-in for `swift test`: records each invocation and appends one + line to the log the gate points it at, the way a real re-run would.""" + script = self.tmp / "remeasure.sh" + body = ["#!/bin/sh", f'echo run >> "{self.invocations}"'] + if append is not None: + body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"") + script.write_text("\n".join(body) + "\n", encoding="utf-8") + script.chmod(0o755) + return script + + def run_gate( + self, + log: Path, + floors: Path, + *, + remeasure: Path | None = None, + attempts: int | None = None, + skip: bool = False, + ) -> subprocess.CompletedProcess: + env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")} + # Never fall through to the default `swift test` re-measure: a test + # that unexpectedly retries must fail fast, not build the app. + env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure()) + if attempts is not None: + env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts) + if skip: + env["BITCHAT_SKIP_PERF_BASELINES"] = "1" + return subprocess.run( + [str(GATE), str(log), str(floors)], + cwd=self.tmp, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def remeasure_count(self) -> int: + if not self.invocations.exists(): + return 0 + return len(self.invocations.read_text(encoding="utf-8").splitlines()) + + def test_every_floor_met_passes(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all benchmarks at or above their floors", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_best_value_across_appended_attempts_is_what_counts(self) -> None: + # Re-measurements append to the same log, so one line clearing the + # floor is enough. The clearing line comes first: a gate that kept the + # last value instead of the best would fail here. + result = self.run_gate( + self.log("PERF[a.one]: 500 ops/sec", "PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + + def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("re-measuring (attempt 2", result.stdout) + + def test_below_floor_on_every_attempt_fails(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"), + attempts=2, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout) + self.assertIn("treating as a real regression", result.stderr) + + def test_missing_floored_benchmark_fails_without_a_retry(self) -> None: + # A re-measurement that happened to produce the missing line would + # paper over a silently dropped benchmark, so this must not retry. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"), + ) + + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout) + + def test_unfloored_benchmark_fails_without_a_retry(self) -> None: + # The mirror of the missing case: a benchmark that measures but has no + # floor guards nothing, and no re-measurement can supply the floor. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertIn("NO-FLOOR c.new: 1000 ops/sec reports a PERF line but has no floor", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_malformed_floors_file_fails_without_a_retry(self) -> None: + # A broken floors file is a setup error, not runner noise: the gate + # must report it at once, without a re-measurement. The PERF line is + # there only so the gate does not skip for "no PERF lines". + cases = { + "invalid JSON": '{"floors": {"a.one": 100,}}', + "top level that is not an object": '[{"floors": {"a.one": 100}}]', + "no floors object": '{"limits": {"a.one": 100}}', + "floor that is not a number": '{"floors": {"a.one": "100"}}', + "floor that is a boolean": '{"floors": {"a.one": true}}', + "floor that is not finite": '{"floors": {"a.one": NaN}}', + } + for case, text in cases.items(): + with self.subTest(case): + self.invocations.unlink(missing_ok=True) + floors = self.tmp / "floors.json" + floors.write_text(text, encoding="utf-8") + result = self.run_gate(self.log("PERF[a.one]: 50 ops/sec"), floors) + + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("malformed floors file", result.stderr) + self.assertEqual(self.remeasure_count(), 0) + + def test_output_without_perf_lines_skips(self) -> None: + # Package-only matrix entries produce no benchmarks; the gate must + # not fail them for it. + result = self.run_gate( + self.log("Test Suite 'All tests' passed", "Executed 12 tests"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("no PERF lines", result.stdout) + + def test_skip_flag_bypasses_the_gate_entirely(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1 ops/sec"), + self.floors(**{"a.one": 100}), + skip=True, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + +if __name__ == "__main__": + unittest.main()