fix(axelera): revert primer-index-drain; it wedged production startup

The primer 'drain by run index' was too fragile under Frigate's real
workload: with 7 cameras starting simultaneously the first few real frames
arrive before the SDK finishes emitting primer results, their ids were
drained without a result, run() hit the timeout and Frigate restarted the
detect process every ~15s (zero detections).

Revert to the stable approach: keep-alive frames sustain the SDK during
quiet/init, _pump never consumes a pending id while the queue is empty, and
run() is serialized (lock) so at most one real frame is in flight. The
remaining theoretical primer/keep-alive misattribution window only exists
when the queue is empty mid-run, which does not happen under Frigate's
continuous feed.
This commit is contained in:
patrykorwat 2026-08-20 21:19:04 +00:00
parent 13215b0997
commit 9d8a1a10a4

View File

@ -153,9 +153,6 @@ class _AxeleraInference:
# flight the FIFO pop can misattribute a result to the wrong camera.
# With at most one frame in flight the result can only belong to it.
self._run_lock = threading.Lock()
# Set once the SDK has finished emitting the primer results, so they
# can never be attributed to a real pending request id.
self._primed = threading.Event()
if stream_factory is None:
# lazy import: never import the SDK unless the detector is used
@ -181,19 +178,21 @@ class _AxeleraInference:
def _feed(self):
# primer frames: the SDK init pre-buffers several frames before the
# source yields (validated on hardware). Their results are drained by
# _pump (by index) and NEVER attributed to a request id.
# source yields (validated on hardware). Their results are dropped by
# the pump while the pending queue is empty.
for _ in range(PRIMER_FRAMES):
yield np.zeros((self._height, self._width, 3), np.uint8)
while True:
frame = frame_id = None
try:
frame_id, frame = self._frame_q.get(timeout=1.0)
except queue.Empty:
# Never synthesize keep-alive frames: a synthetic frame's
# result can arrive while a real id is pending and steal that
# id (misattribution across cameras). In Frigate the queue is
# fed continuously, so idling simply waits here.
continue
# keep-alive: no request pending, keep the SDK stream fed so the
# inference timeout never kills it during quiet periods. The
# None id means this frame's result is dropped by the pump.
frame_id, frame = None, np.zeros(
(self._height, self._width, 3), np.uint8
)
if frame is None: # shutdown sentinel from stop()
return
if frame_id is not None:
@ -255,14 +254,7 @@ class _AxeleraInference:
def _pump(self):
logger.info("axelera: inference stream started")
try:
for run_index, frame_result in enumerate(self._stream):
if run_index < PRIMER_FRAMES:
# Primer results are drained WITHOUT consuming a request
# id — otherwise a buffered primer result could steal a
# real id that is already pending (P1 misattribution).
if run_index == PRIMER_FRAMES - 1:
self._primed.set()
continue
for frame_result in self._stream:
out = _pad_rows(self._detection_rows(frame_result))
if self._pending_ids:
# FIFO: the stream returns results in submission order, so
@ -309,12 +301,6 @@ class _AxeleraInference:
# permanently wedging this detector.
raise RuntimeError("Axelera inference thread stopped; restart required.")
# Wait briefly for the primer results to drain ONCE, so they can never
# be attributed to a real request id. After this the pump only ever
# sees real frames (keep-alive frames are no longer emitted).
if not self._primed.is_set():
self._primed.wait(timeout=0.5)
# Every submitted frame gets a unique id; the pump stores the result
# under this id, so we can wait for OUR frame's outcome exclusively.
self._next_id += 1