From cd8b274f895cf8d2df10663caab7d5adc56f0437 Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Thu, 24 Sep 2026 22:31:49 +0200 Subject: [PATCH 1/5] Pin the performance floor gate's contract with tests check-perf-floors.sh is the only check between an order-of-magnitude regression and main, and it has no tests: scripts/ carries a test file for validate_georelays.py and two for the workflows around it, and nothing for the script that decides whether a slow hot path merges. Its behaviour is a shell retry loop around a Python heredoc that parses captured test output and picks exit codes -- exactly the kind of logic that drifts quietly. This runs the real script against fixture PERF logs and pins the documented contract: - every floor met passes; - the best value across appended attempts is what counts, so one retry that clears the floor is enough; - a below-floor result is re-measured and passes once it clears; - a below-floor result on every attempt fails (exit 1); - a floored benchmark that stops reporting fails without a retry (exit 3) -- a re-measurement that happened to produce the line would paper over a silently dropped benchmark; - no PERF lines and BITCHAT_SKIP_PERF_BASELINES=1 both skip (exit 0). The re-measure command is replaced by a stand-in that records each invocation and appends whatever the test says to the log the gate points it at, so the retry path runs in milliseconds with no toolchain, and a test that retries unexpectedly fails fast instead of building the app. One test pins a behaviour rather than endorsing it: a benchmark that reports a PERF line with no floor prints "consider adding a floor" and passes. That is the gate's one advisory branch -- removing a floored benchmark fails, adding an unfloored one does not -- and it is left as found so the follow-up flipping it is a three-line script change with a one-assertion test diff. The tests run in the app matrix entry immediately before the gate they describe. They also fall under the scripts/tests discovery in validate_georelays.yml, which is harmless. --- .github/workflows/swift-tests.yml | 6 + scripts/tests/test_check_perf_floors.py | 173 ++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 scripts/tests/test_check_perf_floors.py diff --git a/.github/workflows/swift-tests.yml b/.github/workflows/swift-tests.yml index e64683de..b287dfe1 100644 --- a/.github/workflows/swift-tests.yml +++ b/.github/workflows/swift-tests.yml @@ -121,6 +121,12 @@ jobs: BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log run: swift test --quiet --filter PerformanceBaselineTests + # The gate below is the only check between a benchmark regression and + # main; its exit-code contract is pinned by scripts/tests before it runs. + - name: Test the performance floor gate + if: matrix.name == 'app' + run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v + # Order-of-magnitude performance regression gate. Floors are deliberately # generous (see bitchatTests/Performance/perf-floors.json) so this # catches algorithmic regressions, never runner variance. If a metric diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py new file mode 100644 index 00000000..156d7dba --- /dev/null +++ b/scripts/tests/test_check_perf_floors.py @@ -0,0 +1,173 @@ +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh" + + +class CheckPerfFloorsTests(unittest.TestCase): + """The floor gate is what stands between an order-of-magnitude regression + and main, and it decides by parsing captured test output. These run the + real script against fixture logs and pin its exit-code contract: which + outcomes skip, which are re-measured, and which fail without a retry. + """ + + def setUp(self) -> None: + self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-")) + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.invocations = self.tmp / "remeasure-invocations" + + def floors(self, **floors: int) -> Path: + path = self.tmp / "floors.json" + entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items()) + path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8") + return path + + def log(self, *lines: str) -> Path: + path = self.tmp / "perf-output.log" + path.write_text("".join(line + "\n" for line in lines), encoding="utf-8") + return path + + def remeasure(self, append: str | None = None) -> Path: + """A stand-in for `swift test`: records each invocation and appends one + line to the log the gate points it at, the way a real re-run would.""" + script = self.tmp / "remeasure.sh" + body = ["#!/bin/sh", f'echo run >> "{self.invocations}"'] + if append is not None: + body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"") + script.write_text("\n".join(body) + "\n", encoding="utf-8") + script.chmod(0o755) + return script + + def run_gate( + self, + log: Path, + floors: Path, + *, + remeasure: Path | None = None, + attempts: int | None = None, + skip: bool = False, + ) -> subprocess.CompletedProcess: + env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")} + # Never fall through to the default `swift test` re-measure: a test + # that unexpectedly retries must fail fast, not build the app. + env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure()) + if attempts is not None: + env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts) + if skip: + env["BITCHAT_SKIP_PERF_BASELINES"] = "1" + return subprocess.run( + [str(GATE), str(log), str(floors)], + cwd=self.tmp, + env=env, + capture_output=True, + text=True, + check=False, + ) + + def remeasure_count(self) -> int: + if not self.invocations.exists(): + return 0 + return len(self.invocations.read_text(encoding="utf-8").splitlines()) + + def test_every_floor_met_passes(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("all benchmarks at or above their floors", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_best_value_across_appended_attempts_is_what_counts(self) -> None: + # Re-measurements append to the same log, so one line clearing the + # floor is enough even when an earlier attempt sat under it. + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec", "PERF[a.one]: 500 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + + def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("re-measuring (attempt 2", result.stdout) + + def test_below_floor_on_every_attempt_fails(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 50 ops/sec"), + self.floors(**{"a.one": 100}), + remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"), + attempts=2, + ) + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 1) + self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout) + self.assertIn("treating as a real regression", result.stderr) + + def test_missing_floored_benchmark_fails_without_a_retry(self) -> None: + # A re-measurement that happened to produce the missing line would + # paper over a silently dropped benchmark, so this must not retry. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec"), + self.floors(**{"a.one": 100, "b.two": 200}), + remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"), + ) + + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertEqual(self.remeasure_count(), 0) + self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout) + + def test_unfloored_benchmark_is_reported_but_does_not_fail(self) -> None: + # Pinned as found, not endorsed: the gate fails when a floored + # benchmark stops reporting, but a benchmark that reports with no floor + # only prints advice. That asymmetry is the subject of the follow-up. + result = self.run_gate( + self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("NO-FLOOR c.new: 1000 ops/sec (consider adding a floor)", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + def test_output_without_perf_lines_skips(self) -> None: + # Package-only matrix entries produce no benchmarks; the gate must + # not fail them for it. + result = self.run_gate( + self.log("Test Suite 'All tests' passed", "Executed 12 tests"), + self.floors(**{"a.one": 100}), + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("no PERF lines", result.stdout) + + def test_skip_flag_bypasses_the_gate_entirely(self) -> None: + result = self.run_gate( + self.log("PERF[a.one]: 1 ops/sec"), + self.floors(**{"a.one": 100}), + skip=True, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout) + self.assertEqual(self.remeasure_count(), 0) + + +if __name__ == "__main__": + unittest.main() From 217b522ec345d3ed7c8276cb1c2220b9514853c5 Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Thu, 24 Sep 2026 22:34:02 +0200 Subject: [PATCH 2/5] Fail the floor gate on a benchmark with no floor The gate fails when a floored benchmark stops reporting (exit 3, no retry) so that a benchmark cannot be dropped without an explicit floors-file change. The other direction was advisory: a benchmark that reports a PERF line with no floor printed "consider adding a floor" into the job log and passed. Nothing else surfaces that line, so a new hot-path benchmark could measure on every run and guard nothing, and the file the gate reads would still look complete. That is the shape of the localization gap closed in #1654, where a key missing from every catalog compiled and shipped English while CI stayed green, and the move is the one made in #1646 and #1648: an advisory check becomes blocking once its backlog is zero. perf-floors.json carries 12 floors and PerformanceBaselineTests has 12 benchmarks, so flipping this costs nothing today and holds the two in step from now on. An unfloored benchmark now fails alongside a missing one, with the same exit code and for the same reason: the floors file and the benchmark suite must change together, and no re-measurement can repair a drift between them. The message names the file to edit. The test pinned in the previous commit flips from "reported but does not fail" to "fails without a retry"; the philosophy note in the floors file says "added, renamed or removed" instead of "renamed or removed". --- bitchatTests/Performance/perf-floors.json | 5 +++-- scripts/check-perf-floors.sh | 18 +++++++++++------- scripts/tests/test_check_perf_floors.py | 11 +++++------ 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/bitchatTests/Performance/perf-floors.json b/bitchatTests/Performance/perf-floors.json index 751caf74..3fa9b246 100644 --- a/bitchatTests/Performance/perf-floors.json +++ b/bitchatTests/Performance/perf-floors.json @@ -12,8 +12,9 @@ "magnitude regressions while never flaking on a slow runner.", "Raise floors deliberately after intentional performance improvements;", "lower them only with a written justification in the PR. If a benchmark is", - "renamed or removed, update this file in the same change - the gate fails", - "when a floored benchmark stops reporting.", + "added, renamed or removed, update this file in the same change - the gate", + "fails when a floored benchmark stops reporting or when a benchmark reports", + "with no floor.", "Checked by scripts/check-perf-floors.sh against captured swift-test output." ], "_units": "operations per second, matching each benchmark's PERF line", diff --git a/scripts/check-perf-floors.sh b/scripts/check-perf-floors.sh index b51eb390..71e1dc57 100755 --- a/scripts/check-perf-floors.sh +++ b/scripts/check-perf-floors.sh @@ -36,9 +36,10 @@ # Fails when: # - any benchmark reports throughput below its floor on every attempt # (exit 1), or -# - PERF lines are present but a floored benchmark is missing — a -# silently-dropped benchmark must be an explicit floors-file change and -# is not retried (exit 3). +# - PERF lines are present but a floored benchmark is missing, or a +# benchmark reports with no floor — the floors file and the benchmark +# suite must change together in both directions, and neither drift is +# retried (exit 3). set -euo pipefail @@ -111,6 +112,7 @@ with open(output_file, errors="replace") as f: below_floor = [] missing = [] +unfloored = [] print(f"perf-floors: checking {len(measured)} benchmark(s) against {len(floors)} floor(s)") for name in sorted(set(floors) | set(measured)): floor = floors.get(name) @@ -121,7 +123,9 @@ for name in sorted(set(floors) | set(measured)): continue value, unit = measured[name] if floor is None: - print(f" NO-FLOOR {name}: {value:.0f} {unit}/sec (consider adding a floor)") + unfloored.append( + f" NO-FLOOR {name}: {value:.0f} {unit}/sec reports a PERF line but has " + f"no floor (add one to perf-floors.json in the same change)") continue status = "OK" if value >= floor else "BELOW" line = f" {status:8} {name}: {value:.0f} {unit}/sec (floor {floor})" @@ -131,9 +135,9 @@ for name in sorted(set(floors) | set(measured)): f" BELOW {name}: {value:.0f} {unit}/sec is under floor {floor} " f"({value / floor * 100:.0f}% of floor)") -if missing: - print("\nperf-floors: FAILED — floored benchmark(s) missing from the output:") - print("\n".join(missing + below_floor)) +if missing or unfloored: + print("\nperf-floors: FAILED — floors and benchmarks are out of step:") + print("\n".join(missing + unfloored + below_floor)) sys.exit(3) if below_floor: diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py index 156d7dba..ecbdcdf9 100644 --- a/scripts/tests/test_check_perf_floors.py +++ b/scripts/tests/test_check_perf_floors.py @@ -133,17 +133,16 @@ class CheckPerfFloorsTests(unittest.TestCase): self.assertEqual(self.remeasure_count(), 0) self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout) - def test_unfloored_benchmark_is_reported_but_does_not_fail(self) -> None: - # Pinned as found, not endorsed: the gate fails when a floored - # benchmark stops reporting, but a benchmark that reports with no floor - # only prints advice. That asymmetry is the subject of the follow-up. + def test_unfloored_benchmark_fails_without_a_retry(self) -> None: + # The mirror of the missing case: a benchmark that measures but has no + # floor guards nothing, and no re-measurement can supply the floor. result = self.run_gate( self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"), self.floors(**{"a.one": 100}), ) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - self.assertIn("NO-FLOOR c.new: 1000 ops/sec (consider adding a floor)", result.stdout) + self.assertEqual(result.returncode, 3, result.stdout + result.stderr) + self.assertIn("NO-FLOOR c.new: 1000 ops/sec reports a PERF line but has no floor", result.stdout) self.assertEqual(self.remeasure_count(), 0) def test_output_without_perf_lines_skips(self) -> None: From 54e691e4197411f6aa33943076dbac2f55d307ea Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Sun, 1 Nov 2026 17:28:41 +0100 Subject: [PATCH 3/5] Import future annotations so the gate tests load on Python 3.9 The test file annotates with `str | None` and `Path | None`. Python evaluates these when it defines each function, and Python 3.9, the stock macOS python3, has no `|` on types. So the import raised a TypeError, and discovery of the whole scripts/tests folder failed. With the future import, Python keeps the annotations as strings and does not evaluate them. --- scripts/tests/test_check_perf_floors.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py index ecbdcdf9..7fd2db71 100644 --- a/scripts/tests/test_check_perf_floors.py +++ b/scripts/tests/test_check_perf_floors.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import os from pathlib import Path import shutil From a2061b07d38bf038dec0d63312399394cf39a8af Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Sun, 1 Nov 2026 17:29:00 +0100 Subject: [PATCH 4/5] Log the clearing value first in the best-value gate test The test logged 50, then 500. The last value was also the best one, so a gate that kept the last value instead of the best passed all eight tests. With 500 first and 50 last, that gate stays below the floor and fails this test. --- scripts/tests/test_check_perf_floors.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py index 7fd2db71..00f06e8c 100644 --- a/scripts/tests/test_check_perf_floors.py +++ b/scripts/tests/test_check_perf_floors.py @@ -89,9 +89,10 @@ class CheckPerfFloorsTests(unittest.TestCase): def test_best_value_across_appended_attempts_is_what_counts(self) -> None: # Re-measurements append to the same log, so one line clearing the - # floor is enough even when an earlier attempt sat under it. + # floor is enough. The clearing line comes first: a gate that kept the + # last value instead of the best would fail here. result = self.run_gate( - self.log("PERF[a.one]: 50 ops/sec", "PERF[a.one]: 500 ops/sec"), + self.log("PERF[a.one]: 500 ops/sec", "PERF[a.one]: 50 ops/sec"), self.floors(**{"a.one": 100}), ) From 739077305f265e0f2520ba7a9a194d6f90d1f780 Mon Sep 17 00:00:00 2001 From: Jozef Koval Date: Sun, 1 Nov 2026 17:31:35 +0100 Subject: [PATCH 5/5] Exit 2 on a malformed floors file instead of re-measuring A broken floors file raised an uncaught exception, so Python exited 1. The loop read exit 1 as "below floor" and measured twice more before it reported a real regression. The gate now names the file and exits 2. The loop does not retry exit 2, as its comment says for setup errors. Broken means unreadable, not valid JSON, no "floors" object, or a floor that is not a finite number. Some of these did not crash before, but gave a wrong result. A NaN floor let every value pass, because a comparison with NaN is always false. A `true` floor counted as 1, and a `null` floor was reported as NO-FLOOR with exit 3. test_malformed_floors_file_fails_without_a_retry covers six cases. The exit-code comments also called exit 3 "floored benchmark missing". This branch fails unfloored benchmarks with exit 3 too, so they now say "floors and benchmarks out of step". --- scripts/check-perf-floors.sh | 29 +++++++++++++++++++++---- scripts/tests/test_check_perf_floors.py | 23 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/scripts/check-perf-floors.sh b/scripts/check-perf-floors.sh index 71e1dc57..8e567418 100755 --- a/scripts/check-perf-floors.sh +++ b/scripts/check-perf-floors.sh @@ -36,6 +36,8 @@ # Fails when: # - any benchmark reports throughput below its floor on every attempt # (exit 1), or +# - the floors file is missing, unreadable or malformed — a setup error +# that no re-measurement can fix, so it is not retried (exit 2), or # - PERF lines are present but a floored benchmark is missing, or a # benchmark reports with no floor — the floors file and the benchmark # suite must change together in both directions, and neither drift is @@ -81,10 +83,12 @@ case "$OUTPUT_FILE" in esac # Exit codes: 0 = all floors met, 1 = below floor (retryable — noise vs -# regression undecided), 3 = floored benchmark missing (not retryable). +# regression undecided), 2 = unreadable or malformed floors file, 3 = floors +# and benchmarks out of step (neither retryable). check_floors() { OUTPUT_FILE="$OUTPUT_FILE" FLOORS_FILE="$FLOORS_FILE" python3 - <<'PYEOF' import json +import math import os import re import sys @@ -92,8 +96,24 @@ import sys output_file = os.environ["OUTPUT_FILE"] floors_file = os.environ["FLOORS_FILE"] -with open(floors_file) as f: - floors = json.load(f)["floors"] +def malformed(detail): + # A setup error, not runner noise: exit 2 so the loop does not re-measure. + print(f"perf-floors: unreadable or malformed floors file '{floors_file}': {detail}", file=sys.stderr) + sys.exit(2) + +try: + with open(floors_file) as f: + document = json.load(f) +except (OSError, ValueError) as error: + malformed(error) +floors = document.get("floors") if isinstance(document, dict) else None +if not isinstance(floors, dict): + malformed('it needs a "floors" object') +for name, floor in floors.items(): + # NaN would fail every comparison, so a below-floor value would pass. + is_number = isinstance(floor, (int, float)) and not isinstance(floor, bool) + if not is_number or not math.isfinite(floor): + malformed(f'the floor for "{name}" is not a finite number') # PERF[delivery.storeUpdate]: 158862 updates/sec (avg 3.147 ms per pass of 500, 10 passes) pattern = re.compile(r"PERF\[([^\]]+)\]:\s*([0-9]+(?:\.[0-9]+)?)\s*(\S+)/sec") @@ -165,7 +185,8 @@ while true; do # Below floor: retry to separate runner noise from regression. ;; *) - # Missing benchmark or parse/setup error: re-measuring can't help. + # Floors and benchmarks out of step, or a setup error such as a + # malformed floors file: re-measuring can't help. exit "$gate_status" ;; esac diff --git a/scripts/tests/test_check_perf_floors.py b/scripts/tests/test_check_perf_floors.py index 00f06e8c..92bb07cd 100644 --- a/scripts/tests/test_check_perf_floors.py +++ b/scripts/tests/test_check_perf_floors.py @@ -148,6 +148,29 @@ class CheckPerfFloorsTests(unittest.TestCase): self.assertIn("NO-FLOOR c.new: 1000 ops/sec reports a PERF line but has no floor", result.stdout) self.assertEqual(self.remeasure_count(), 0) + def test_malformed_floors_file_fails_without_a_retry(self) -> None: + # A broken floors file is a setup error, not runner noise: the gate + # must report it at once, without a re-measurement. The PERF line is + # there only so the gate does not skip for "no PERF lines". + cases = { + "invalid JSON": '{"floors": {"a.one": 100,}}', + "top level that is not an object": '[{"floors": {"a.one": 100}}]', + "no floors object": '{"limits": {"a.one": 100}}', + "floor that is not a number": '{"floors": {"a.one": "100"}}', + "floor that is a boolean": '{"floors": {"a.one": true}}', + "floor that is not finite": '{"floors": {"a.one": NaN}}', + } + for case, text in cases.items(): + with self.subTest(case): + self.invocations.unlink(missing_ok=True) + floors = self.tmp / "floors.json" + floors.write_text(text, encoding="utf-8") + result = self.run_gate(self.log("PERF[a.one]: 50 ops/sec"), floors) + + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn("malformed floors file", result.stderr) + self.assertEqual(self.remeasure_count(), 0) + def test_output_without_perf_lines_skips(self) -> None: # Package-only matrix entries produce no benchmarks; the gate must # not fail them for it.