Pin the performance floor gate's contract with tests

check-perf-floors.sh is the only check between an order-of-magnitude
regression and main, and it has no tests: scripts/ carries a test file
for validate_georelays.py and two for the workflows around it, and
nothing for the script that decides whether a slow hot path merges. Its
behaviour is a shell retry loop around a Python heredoc that parses
captured test output and picks exit codes -- exactly the kind of logic
that drifts quietly.

This runs the real script against fixture PERF logs and pins the
documented contract:

- every floor met passes;
- the best value across appended attempts is what counts, so one
  retry that clears the floor is enough;
- a below-floor result is re-measured and passes once it clears;
- a below-floor result on every attempt fails (exit 1);
- a floored benchmark that stops reporting fails without a retry
  (exit 3) -- a re-measurement that happened to produce the line would
  paper over a silently dropped benchmark;
- no PERF lines and BITCHAT_SKIP_PERF_BASELINES=1 both skip (exit 0).

The re-measure command is replaced by a stand-in that records each
invocation and appends whatever the test says to the log the gate
points it at, so the retry path runs in milliseconds with no toolchain,
and a test that retries unexpectedly fails fast instead of building the
app.

One test pins a behaviour rather than endorsing it: a benchmark that
reports a PERF line with no floor prints "consider adding a floor" and
passes. That is the gate's one advisory branch -- removing a floored
benchmark fails, adding an unfloored one does not -- and it is left as
found so the follow-up flipping it is a three-line script change with a
one-assertion test diff.

The tests run in the app matrix entry immediately before the gate they
describe. They also fall under the scripts/tests discovery in
validate_georelays.yml, which is harmless.
This commit is contained in:
Jozef Koval 2026-09-24 22:31:49 +02:00
parent 5e9287fae1
commit cd8b274f89
2 changed files with 179 additions and 0 deletions

View File

@ -121,6 +121,12 @@ jobs:
BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log
run: swift test --quiet --filter PerformanceBaselineTests
# The gate below is the only check between a benchmark regression and
# main; its exit-code contract is pinned by scripts/tests before it runs.
- name: Test the performance floor gate
if: matrix.name == 'app'
run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v
# Order-of-magnitude performance regression gate. Floors are deliberately
# generous (see bitchatTests/Performance/perf-floors.json) so this
# catches algorithmic regressions, never runner variance. If a metric

View File

@ -0,0 +1,173 @@
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh"
class CheckPerfFloorsTests(unittest.TestCase):
"""The floor gate is what stands between an order-of-magnitude regression
and main, and it decides by parsing captured test output. These run the
real script against fixture logs and pin its exit-code contract: which
outcomes skip, which are re-measured, and which fail without a retry.
"""
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-"))
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
self.invocations = self.tmp / "remeasure-invocations"
def floors(self, **floors: int) -> Path:
path = self.tmp / "floors.json"
entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items())
path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8")
return path
def log(self, *lines: str) -> Path:
path = self.tmp / "perf-output.log"
path.write_text("".join(line + "\n" for line in lines), encoding="utf-8")
return path
def remeasure(self, append: str | None = None) -> Path:
"""A stand-in for `swift test`: records each invocation and appends one
line to the log the gate points it at, the way a real re-run would."""
script = self.tmp / "remeasure.sh"
body = ["#!/bin/sh", f'echo run >> "{self.invocations}"']
if append is not None:
body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"")
script.write_text("\n".join(body) + "\n", encoding="utf-8")
script.chmod(0o755)
return script
def run_gate(
self,
log: Path,
floors: Path,
*,
remeasure: Path | None = None,
attempts: int | None = None,
skip: bool = False,
) -> subprocess.CompletedProcess:
env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")}
# Never fall through to the default `swift test` re-measure: a test
# that unexpectedly retries must fail fast, not build the app.
env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure())
if attempts is not None:
env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts)
if skip:
env["BITCHAT_SKIP_PERF_BASELINES"] = "1"
return subprocess.run(
[str(GATE), str(log), str(floors)],
cwd=self.tmp,
env=env,
capture_output=True,
text=True,
check=False,
)
def remeasure_count(self) -> int:
if not self.invocations.exists():
return 0
return len(self.invocations.read_text(encoding="utf-8").splitlines())
def test_every_floor_met_passes(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"),
self.floors(**{"a.one": 100, "b.two": 200}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("all benchmarks at or above their floors", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
def test_best_value_across_appended_attempts_is_what_counts(self) -> None:
# Re-measurements append to the same log, so one line clearing the
# floor is enough even when an earlier attempt sat under it.
result = self.run_gate(
self.log("PERF[a.one]: 50 ops/sec", "PERF[a.one]: 500 ops/sec"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 0)
def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 50 ops/sec"),
self.floors(**{"a.one": 100}),
remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 1)
self.assertIn("re-measuring (attempt 2", result.stdout)
def test_below_floor_on_every_attempt_fails(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 50 ops/sec"),
self.floors(**{"a.one": 100}),
remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"),
attempts=2,
)
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 1)
self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout)
self.assertIn("treating as a real regression", result.stderr)
def test_missing_floored_benchmark_fails_without_a_retry(self) -> None:
# A re-measurement that happened to produce the missing line would
# paper over a silently dropped benchmark, so this must not retry.
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec"),
self.floors(**{"a.one": 100, "b.two": 200}),
remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"),
)
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 0)
self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout)
def test_unfloored_benchmark_is_reported_but_does_not_fail(self) -> None:
# Pinned as found, not endorsed: the gate fails when a floored
# benchmark stops reporting, but a benchmark that reports with no floor
# only prints advice. That asymmetry is the subject of the follow-up.
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("NO-FLOOR c.new: 1000 ops/sec (consider adding a floor)", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
def test_output_without_perf_lines_skips(self) -> None:
# Package-only matrix entries produce no benchmarks; the gate must
# not fail them for it.
result = self.run_gate(
self.log("Test Suite 'All tests' passed", "Executed 12 tests"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("no PERF lines", result.stdout)
def test_skip_flag_bypasses_the_gate_entirely(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 1 ops/sec"),
self.floors(**{"a.one": 100}),
skip=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
if __name__ == "__main__":
unittest.main()