mirror of
https://github.com/permissionlesstech/bitchat.git
synced 2026-11-15 08:12:23 +00:00
Merge 739077305f265e0f2520ba7a9a194d6f90d1f780 into 04995f68e470a18d1fc99f86fa326c2a87bab4f2
This commit is contained in:
commit
e1ed7213da
6
.github/workflows/swift-tests.yml
vendored
6
.github/workflows/swift-tests.yml
vendored
@ -121,6 +121,12 @@ jobs:
|
||||
BITCHAT_PERF_LOG: ${{ github.workspace }}/perf-output.log
|
||||
run: swift test --quiet --filter PerformanceBaselineTests
|
||||
|
||||
# The gate below is the only check between a benchmark regression and
|
||||
# main; its exit-code contract is pinned by scripts/tests before it runs.
|
||||
- name: Test the performance floor gate
|
||||
if: matrix.name == 'app'
|
||||
run: python3 -m unittest discover -s scripts/tests -p "test_check_perf_floors.py" -v
|
||||
|
||||
# Order-of-magnitude performance regression gate. Floors are deliberately
|
||||
# generous (see bitchatTests/Performance/perf-floors.json) so this
|
||||
# catches algorithmic regressions, never runner variance. If a metric
|
||||
|
||||
@ -12,8 +12,9 @@
|
||||
"magnitude regressions while never flaking on a slow runner.",
|
||||
"Raise floors deliberately after intentional performance improvements;",
|
||||
"lower them only with a written justification in the PR. If a benchmark is",
|
||||
"renamed or removed, update this file in the same change - the gate fails",
|
||||
"when a floored benchmark stops reporting.",
|
||||
"added, renamed or removed, update this file in the same change - the gate",
|
||||
"fails when a floored benchmark stops reporting or when a benchmark reports",
|
||||
"with no floor.",
|
||||
"Checked by scripts/check-perf-floors.sh against captured swift-test output."
|
||||
],
|
||||
"_units": "operations per second, matching each benchmark's PERF line",
|
||||
|
||||
@ -36,9 +36,12 @@
|
||||
# Fails when:
|
||||
# - any benchmark reports throughput below its floor on every attempt
|
||||
# (exit 1), or
|
||||
# - PERF lines are present but a floored benchmark is missing — a
|
||||
# silently-dropped benchmark must be an explicit floors-file change and
|
||||
# is not retried (exit 3).
|
||||
# - the floors file is missing, unreadable or malformed — a setup error
|
||||
# that no re-measurement can fix, so it is not retried (exit 2), or
|
||||
# - PERF lines are present but a floored benchmark is missing, or a
|
||||
# benchmark reports with no floor — the floors file and the benchmark
|
||||
# suite must change together in both directions, and neither drift is
|
||||
# retried (exit 3).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@ -80,10 +83,12 @@ case "$OUTPUT_FILE" in
|
||||
esac
|
||||
|
||||
# Exit codes: 0 = all floors met, 1 = below floor (retryable — noise vs
|
||||
# regression undecided), 3 = floored benchmark missing (not retryable).
|
||||
# regression undecided), 2 = unreadable or malformed floors file, 3 = floors
|
||||
# and benchmarks out of step (neither retryable).
|
||||
check_floors() {
|
||||
OUTPUT_FILE="$OUTPUT_FILE" FLOORS_FILE="$FLOORS_FILE" python3 - <<'PYEOF'
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
@ -91,8 +96,24 @@ import sys
|
||||
output_file = os.environ["OUTPUT_FILE"]
|
||||
floors_file = os.environ["FLOORS_FILE"]
|
||||
|
||||
with open(floors_file) as f:
|
||||
floors = json.load(f)["floors"]
|
||||
def malformed(detail):
|
||||
# A setup error, not runner noise: exit 2 so the loop does not re-measure.
|
||||
print(f"perf-floors: unreadable or malformed floors file '{floors_file}': {detail}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
try:
|
||||
with open(floors_file) as f:
|
||||
document = json.load(f)
|
||||
except (OSError, ValueError) as error:
|
||||
malformed(error)
|
||||
floors = document.get("floors") if isinstance(document, dict) else None
|
||||
if not isinstance(floors, dict):
|
||||
malformed('it needs a "floors" object')
|
||||
for name, floor in floors.items():
|
||||
# NaN would fail every comparison, so a below-floor value would pass.
|
||||
is_number = isinstance(floor, (int, float)) and not isinstance(floor, bool)
|
||||
if not is_number or not math.isfinite(floor):
|
||||
malformed(f'the floor for "{name}" is not a finite number')
|
||||
|
||||
# PERF[delivery.storeUpdate]: 158862 updates/sec (avg 3.147 ms per pass of 500, 10 passes)
|
||||
pattern = re.compile(r"PERF\[([^\]]+)\]:\s*([0-9]+(?:\.[0-9]+)?)\s*(\S+)/sec")
|
||||
@ -111,6 +132,7 @@ with open(output_file, errors="replace") as f:
|
||||
|
||||
below_floor = []
|
||||
missing = []
|
||||
unfloored = []
|
||||
print(f"perf-floors: checking {len(measured)} benchmark(s) against {len(floors)} floor(s)")
|
||||
for name in sorted(set(floors) | set(measured)):
|
||||
floor = floors.get(name)
|
||||
@ -121,7 +143,9 @@ for name in sorted(set(floors) | set(measured)):
|
||||
continue
|
||||
value, unit = measured[name]
|
||||
if floor is None:
|
||||
print(f" NO-FLOOR {name}: {value:.0f} {unit}/sec (consider adding a floor)")
|
||||
unfloored.append(
|
||||
f" NO-FLOOR {name}: {value:.0f} {unit}/sec reports a PERF line but has "
|
||||
f"no floor (add one to perf-floors.json in the same change)")
|
||||
continue
|
||||
status = "OK" if value >= floor else "BELOW"
|
||||
line = f" {status:8} {name}: {value:.0f} {unit}/sec (floor {floor})"
|
||||
@ -131,9 +155,9 @@ for name in sorted(set(floors) | set(measured)):
|
||||
f" BELOW {name}: {value:.0f} {unit}/sec is under floor {floor} "
|
||||
f"({value / floor * 100:.0f}% of floor)")
|
||||
|
||||
if missing:
|
||||
print("\nperf-floors: FAILED — floored benchmark(s) missing from the output:")
|
||||
print("\n".join(missing + below_floor))
|
||||
if missing or unfloored:
|
||||
print("\nperf-floors: FAILED — floors and benchmarks are out of step:")
|
||||
print("\n".join(missing + unfloored + below_floor))
|
||||
sys.exit(3)
|
||||
|
||||
if below_floor:
|
||||
@ -161,7 +185,8 @@ while true; do
|
||||
# Below floor: retry to separate runner noise from regression.
|
||||
;;
|
||||
*)
|
||||
# Missing benchmark or parse/setup error: re-measuring can't help.
|
||||
# Floors and benchmarks out of step, or a setup error such as a
|
||||
# malformed floors file: re-measuring can't help.
|
||||
exit "$gate_status"
|
||||
;;
|
||||
esac
|
||||
|
||||
198
scripts/tests/test_check_perf_floors.py
Normal file
198
scripts/tests/test_check_perf_floors.py
Normal file
@ -0,0 +1,198 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
GATE = REPOSITORY_ROOT / "scripts/check-perf-floors.sh"
|
||||
|
||||
|
||||
class CheckPerfFloorsTests(unittest.TestCase):
|
||||
"""The floor gate is what stands between an order-of-magnitude regression
|
||||
and main, and it decides by parsing captured test output. These run the
|
||||
real script against fixture logs and pin its exit-code contract: which
|
||||
outcomes skip, which are re-measured, and which fail without a retry.
|
||||
"""
|
||||
|
||||
def setUp(self) -> None:
|
||||
self.tmp = Path(tempfile.mkdtemp(prefix="perf-floors-"))
|
||||
self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True)
|
||||
self.invocations = self.tmp / "remeasure-invocations"
|
||||
|
||||
def floors(self, **floors: int) -> Path:
|
||||
path = self.tmp / "floors.json"
|
||||
entries = ",\n".join(f' "{name}": {floor}' for name, floor in floors.items())
|
||||
path.write_text('{\n "floors": {\n' + entries + "\n }\n}\n", encoding="utf-8")
|
||||
return path
|
||||
|
||||
def log(self, *lines: str) -> Path:
|
||||
path = self.tmp / "perf-output.log"
|
||||
path.write_text("".join(line + "\n" for line in lines), encoding="utf-8")
|
||||
return path
|
||||
|
||||
def remeasure(self, append: str | None = None) -> Path:
|
||||
"""A stand-in for `swift test`: records each invocation and appends one
|
||||
line to the log the gate points it at, the way a real re-run would."""
|
||||
script = self.tmp / "remeasure.sh"
|
||||
body = ["#!/bin/sh", f'echo run >> "{self.invocations}"']
|
||||
if append is not None:
|
||||
body.append(f"printf '%s\\n' '{append}' >> \"$BITCHAT_PERF_LOG\"")
|
||||
script.write_text("\n".join(body) + "\n", encoding="utf-8")
|
||||
script.chmod(0o755)
|
||||
return script
|
||||
|
||||
def run_gate(
|
||||
self,
|
||||
log: Path,
|
||||
floors: Path,
|
||||
*,
|
||||
remeasure: Path | None = None,
|
||||
attempts: int | None = None,
|
||||
skip: bool = False,
|
||||
) -> subprocess.CompletedProcess:
|
||||
env = {k: v for k, v in os.environ.items() if not k.startswith("BITCHAT_")}
|
||||
# Never fall through to the default `swift test` re-measure: a test
|
||||
# that unexpectedly retries must fail fast, not build the app.
|
||||
env["BITCHAT_PERF_REMEASURE_CMD"] = str(remeasure or self.remeasure())
|
||||
if attempts is not None:
|
||||
env["BITCHAT_PERF_GATE_ATTEMPTS"] = str(attempts)
|
||||
if skip:
|
||||
env["BITCHAT_SKIP_PERF_BASELINES"] = "1"
|
||||
return subprocess.run(
|
||||
[str(GATE), str(log), str(floors)],
|
||||
cwd=self.tmp,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
def remeasure_count(self) -> int:
|
||||
if not self.invocations.exists():
|
||||
return 0
|
||||
return len(self.invocations.read_text(encoding="utf-8").splitlines())
|
||||
|
||||
def test_every_floor_met_passes(self) -> None:
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 1000 ops/sec (avg 1 ms)", "PERF[b.two]: 250.5 items/sec"),
|
||||
self.floors(**{"a.one": 100, "b.two": 200}),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertIn("all benchmarks at or above their floors", result.stdout)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
|
||||
def test_best_value_across_appended_attempts_is_what_counts(self) -> None:
|
||||
# Re-measurements append to the same log, so one line clearing the
|
||||
# floor is enough. The clearing line comes first: a gate that kept the
|
||||
# last value instead of the best would fail here.
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 500 ops/sec", "PERF[a.one]: 50 ops/sec"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
|
||||
def test_below_floor_is_remeasured_and_passes_once_it_clears(self) -> None:
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 50 ops/sec"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
remeasure=self.remeasure(append="PERF[a.one]: 500 ops/sec"),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual(self.remeasure_count(), 1)
|
||||
self.assertIn("re-measuring (attempt 2", result.stdout)
|
||||
|
||||
def test_below_floor_on_every_attempt_fails(self) -> None:
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 50 ops/sec"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
remeasure=self.remeasure(append="PERF[a.one]: 60 ops/sec"),
|
||||
attempts=2,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
self.assertEqual(self.remeasure_count(), 1)
|
||||
self.assertIn("BELOW a.one: 60 ops/sec is under floor 100", result.stdout)
|
||||
self.assertIn("treating as a real regression", result.stderr)
|
||||
|
||||
def test_missing_floored_benchmark_fails_without_a_retry(self) -> None:
|
||||
# A re-measurement that happened to produce the missing line would
|
||||
# paper over a silently dropped benchmark, so this must not retry.
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 1000 ops/sec"),
|
||||
self.floors(**{"a.one": 100, "b.two": 200}),
|
||||
remeasure=self.remeasure(append="PERF[b.two]: 900 ops/sec"),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
self.assertIn("MISSING b.two: floored benchmark reported no PERF line", result.stdout)
|
||||
|
||||
def test_unfloored_benchmark_fails_without_a_retry(self) -> None:
|
||||
# The mirror of the missing case: a benchmark that measures but has no
|
||||
# floor guards nothing, and no re-measurement can supply the floor.
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 1000 ops/sec", "PERF[c.new]: 1000 ops/sec"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 3, result.stdout + result.stderr)
|
||||
self.assertIn("NO-FLOOR c.new: 1000 ops/sec reports a PERF line but has no floor", result.stdout)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
|
||||
def test_malformed_floors_file_fails_without_a_retry(self) -> None:
|
||||
# A broken floors file is a setup error, not runner noise: the gate
|
||||
# must report it at once, without a re-measurement. The PERF line is
|
||||
# there only so the gate does not skip for "no PERF lines".
|
||||
cases = {
|
||||
"invalid JSON": '{"floors": {"a.one": 100,}}',
|
||||
"top level that is not an object": '[{"floors": {"a.one": 100}}]',
|
||||
"no floors object": '{"limits": {"a.one": 100}}',
|
||||
"floor that is not a number": '{"floors": {"a.one": "100"}}',
|
||||
"floor that is a boolean": '{"floors": {"a.one": true}}',
|
||||
"floor that is not finite": '{"floors": {"a.one": NaN}}',
|
||||
}
|
||||
for case, text in cases.items():
|
||||
with self.subTest(case):
|
||||
self.invocations.unlink(missing_ok=True)
|
||||
floors = self.tmp / "floors.json"
|
||||
floors.write_text(text, encoding="utf-8")
|
||||
result = self.run_gate(self.log("PERF[a.one]: 50 ops/sec"), floors)
|
||||
|
||||
self.assertEqual(result.returncode, 2, result.stdout + result.stderr)
|
||||
self.assertIn("malformed floors file", result.stderr)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
|
||||
def test_output_without_perf_lines_skips(self) -> None:
|
||||
# Package-only matrix entries produce no benchmarks; the gate must
|
||||
# not fail them for it.
|
||||
result = self.run_gate(
|
||||
self.log("Test Suite 'All tests' passed", "Executed 12 tests"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertIn("no PERF lines", result.stdout)
|
||||
|
||||
def test_skip_flag_bypasses_the_gate_entirely(self) -> None:
|
||||
result = self.run_gate(
|
||||
self.log("PERF[a.one]: 1 ops/sec"),
|
||||
self.floors(**{"a.one": 100}),
|
||||
skip=True,
|
||||
)
|
||||
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertIn("BITCHAT_SKIP_PERF_BASELINES=1", result.stdout)
|
||||
self.assertEqual(self.remeasure_count(), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Loading…
x
Reference in New Issue
Block a user