Merge 739077305f265e0f2520ba7a9a194d6f90d1f780 into 04995f68e470a18d1fc99f86fa326c2a87bab4f2

This commit is contained in:
Jozef Koval 2026-11-01 17:13:39 +00:00 committed by GitHub
commit e1ed7213da
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 243 additions and 13 deletions

View File

@ -121,6 +121,12 @@ jobs:
BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log
run: swift test --quiet --filter PerformanceBaselineTests
# The gate below is the only check between a benchmark regression and
# main; its exit-code contract is pinned by scripts/tests before it runs.
- name: Test the performance floor gate
if: matrix.name == 'app'
run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v
# Order-of-magnitude performance regression gate. Floors are deliberately
# generous (see bitchatTests/Performance/perf-floors.json) so this
# catches algorithmic regressions, never runner variance. If a metric

View File

@ -12,8 +12,9 @@
"magnitude regressions while never flaking on a slow runner.",
"Raise floors deliberately after intentional performance improvements;",
"lower them only with a written justification in the PR. If a benchmark is",
"renamed or removed, update this file in the same change - the gate fails",
"when a floored benchmark stops reporting.",
"added, renamed or removed, update this file in the same change - the gate",
"fails when a floored benchmark stops reporting or when a benchmark reports",
"with no floor.",
"Checked by scripts/check-perf-floors.sh against captured swift-test output."
],
"_units": "operations per second, matching each benchmark's PERF line",

View File

@ -36,9 +36,12 @@
# Fails when:
# - any benchmark reports throughput below its floor on every attempt
# (exit 1), or
# - PERF lines are present but a floored benchmark is missing — a
# silently-dropped benchmark must be an explicit floors-file change and
# is not retried (exit 3).
# - the floors file is missing, unreadable or malformed — a setup error
# that no re-measurement can fix, so it is not retried (exit 2), or
# - PERF lines are present but a floored benchmark is missing, or a
# benchmark reports with no floor — the floors file and the benchmark
# suite must change together in both directions, and neither drift is
# retried (exit 3).
set -euo pipefail
@ -80,10 +83,12 @@ case "$OUTPUT_FILE" in
esac
# Exit codes: 0 = all floors met, 1 = below floor (retryable — noise vs
# regression undecided), 3 = floored benchmark missing (not retryable).
# regression undecided), 2 = unreadable or malformed floors file, 3 = floors
# and benchmarks out of step (neither retryable).
check_floors() {
OUTPUT_FILE="$OUTPUT_FILE" FLOORS_FILE="$FLOORS_FILE" python3 - <<'PYEOF'
import json
import math
import os
import re
import sys
@ -91,8 +96,24 @@ import sys
output_file = os.environ["OUTPUT_FILE"]
floors_file = os.environ["FLOORS_FILE"]
with open(floors_file) as f:
floors = json.load(f)["floors"]
def malformed(detail):
# A setup error, not runner noise: exit 2 so the loop does not re-measure.
print(f"perf-floors: unreadable or malformed floors file '{floors_file}': {detail}", file=sys.stderr)
sys.exit(2)
try:
with open(floors_file) as f:
document = json.load(f)
except (OSError, ValueError) as error:
malformed(error)
floors = document.get("floors") if isinstance(document, dict) else None
if not isinstance(floors, dict):
malformed('it needs a "floors" object')
for name, floor in floors.items():
# NaN would fail every comparison, so a below-floor value would pass.
is_number = isinstance(floor, (int, float)) and not isinstance(floor, bool)
if not is_number or not math.isfinite(floor):
malformed(f'the floor for "{name}" is not a finite number')
# PERF[delivery.storeUpdate]: 158862 updates/sec (avg 3.147 ms per pass of 500, 10 passes)
pattern = re.compile(r"PERF\[([^\]]+)\]:\s*([0-9]+(?:\.[0-9]+)?)\s*(\S+)/sec")
@ -111,6 +132,7 @@ with open(output_file, errors="replace") as f:
below_floor = []
missing = []
unfloored = []
print(f"perf-floors: checking {len(measured)} benchmark(s) against {len(floors)} floor(s)")
for name in sorted(set(floors) | set(measured)):
floor = floors.get(name)
@ -121,7 +143,9 @@ for name in sorted(set(floors) | set(measured)):
continue
value, unit = measured[name]
if floor is None:
print(f" NO-FLOOR {name}: {value:.0f} {unit}/sec (consider adding a floor)")
unfloored.append(
f" NO-FLOOR {name}: {value:.0f} {unit}/sec reports a PERF line but has "
f"no floor (add one to perf-floors.json in the same change)")
continue
status = "OK" if value >= floor else "BELOW"
line = f" {status:8} {name}: {value:.0f} {unit}/sec (floor {floor})"
@ -131,9 +155,9 @@ for name in sorted(set(floors) | set(measured)):
f" BELOW {name}: {value:.0f} {unit}/sec is under floor {floor} "
f"({value / floor * 100:.0f}% of floor)")
if missing:
print("\nperf-floors: FAILED — floored benchmark(s) missing from the output:")
print("\n".join(missing + below_floor))
if missing or unfloored:
print("\nperf-floors: FAILED — floors and benchmarks are out of step:")
print("\n".join(missing + unfloored + below_floor))
sys.exit(3)
if below_floor:
@ -161,7 +185,8 @@ while true; do
# Below floor: retry to separate runner noise from regression.
;;
*)
# Missing benchmark or parse/setup error: re-measuring can't help.
# Floors and benchmarks out of step, or a setup error such as a
# malformed floors file: re-measuring can't help.
exit "$gate_status"
;;
esac

View File

@ -0,0 +1,198 @@
from __future__ import annotations
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh"
class CheckPerfFloorsTests(unittest.TestCase):
"""The floor gate is what stands between an order-of-magnitude regression
and main, and it decides by parsing captured test output. These run the
real script against fixture logs and pin its exit-code contract: which
outcomes skip, which are re-measured, and which fail without a retry.
"""
def setUp(self) -> None:
self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-"))
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
self.invocations = self.tmp / "remeasure-invocations"
def floors(self, **floors: int) -> Path:
path = self.tmp / "floors.json"
entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items())
path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8")
return path
def log(self, *lines: str) -> Path:
path = self.tmp / "perf-output.log"
path.write_text("".join(line + "\n" for line in lines), encoding="utf-8")
return path
def remeasure(self, append: str | None = None) -> Path:
"""A stand-in for `swift test`: records each invocation and appends one
line to the log the gate points it at, the way a real re-run would."""
script = self.tmp / "remeasure.sh"
body = ["#!/bin/sh", f'echo run >> "{self.invocations}"']
if append is not None:
body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"")
script.write_text("\n".join(body) + "\n", encoding="utf-8")
script.chmod(0o755)
return script
def run_gate(
self,
log: Path,
floors: Path,
*,
remeasure: Path | None = None,
attempts: int | None = None,
skip: bool = False,
) -> subprocess.CompletedProcess:
env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")}
# Never fall through to the default `swift test` re-measure: a test
# that unexpectedly retries must fail fast, not build the app.
env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure())
if attempts is not None:
env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts)
if skip:
env["BITCHAT_SKIP_PERF_BASELINES"] = "1"
return subprocess.run(
[str(GATE), str(log), str(floors)],
cwd=self.tmp,
env=env,
capture_output=True,
text=True,
check=False,
)
def remeasure_count(self) -> int:
if not self.invocations.exists():
return 0
return len(self.invocations.read_text(encoding="utf-8").splitlines())
def test_every_floor_met_passes(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"),
self.floors(**{"a.one": 100, "b.two": 200}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("all benchmarks at or above their floors", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
def test_best_value_across_appended_attempts_is_what_counts(self) -> None:
# Re-measurements append to the same log, so one line clearing the
# floor is enough. The clearing line comes first: a gate that kept the
# last value instead of the best would fail here.
result = self.run_gate(
self.log("PERF[a.one]: 500 ops/sec", "PERF[a.one]: 50 ops/sec"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 0)
def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 50 ops/sec"),
self.floors(**{"a.one": 100}),
remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 1)
self.assertIn("re-measuring (attempt 2", result.stdout)
def test_below_floor_on_every_attempt_fails(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 50 ops/sec"),
self.floors(**{"a.one": 100}),
remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"),
attempts=2,
)
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 1)
self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout)
self.assertIn("treating as a real regression", result.stderr)
def test_missing_floored_benchmark_fails_without_a_retry(self) -> None:
# A re-measurement that happened to produce the missing line would
# paper over a silently dropped benchmark, so this must not retry.
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec"),
self.floors(**{"a.one": 100, "b.two": 200}),
remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"),
)
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertEqual(self.remeasure_count(), 0)
self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout)
def test_unfloored_benchmark_fails_without_a_retry(self) -> None:
# The mirror of the missing case: a benchmark that measures but has no
# floor guards nothing, and no re-measurement can supply the floor.
result = self.run_gate(
self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
self.assertIn("NO-FLOOR c.new: 1000 ops/sec reports a PERF line but has no floor", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
def test_malformed_floors_file_fails_without_a_retry(self) -> None:
# A broken floors file is a setup error, not runner noise: the gate
# must report it at once, without a re-measurement. The PERF line is
# there only so the gate does not skip for "no PERF lines".
cases = {
"invalid JSON": '{"floors": {"a.one": 100,}}',
"top level that is not an object": '[{"floors": {"a.one": 100}}]',
"no floors object": '{"limits": {"a.one": 100}}',
"floor that is not a number": '{"floors": {"a.one": "100"}}',
"floor that is a boolean": '{"floors": {"a.one": true}}',
"floor that is not finite": '{"floors": {"a.one": NaN}}',
}
for case, text in cases.items():
with self.subTest(case):
self.invocations.unlink(missing_ok=True)
floors = self.tmp / "floors.json"
floors.write_text(text, encoding="utf-8")
result = self.run_gate(self.log("PERF[a.one]: 50 ops/sec"), floors)
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
self.assertIn("malformed floors file", result.stderr)
self.assertEqual(self.remeasure_count(), 0)
def test_output_without_perf_lines_skips(self) -> None:
# Package-only matrix entries produce no benchmarks; the gate must
# not fail them for it.
result = self.run_gate(
self.log("Test Suite 'All tests' passed", "Executed 12 tests"),
self.floors(**{"a.one": 100}),
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("no PERF lines", result.stdout)
def test_skip_flag_bypasses_the_gate_entirely(self) -> None:
result = self.run_gate(
self.log("PERF[a.one]: 1 ops/sec"),
self.floors(**{"a.one": 100}),
skip=True,
)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout)
self.assertEqual(self.remeasure_count(), 0)
if __name__ == "__main__":
unittest.main()