From 92feefc113ca9fcb8c6d9d8e2a3c25730ee50208 Mon Sep 17 00:00:00 2001
From: patrykorwat <patrykorwat@gmail.com>
Date: Sun, 23 Aug 2026 13:40:39 +0000
Subject: [PATCH] fix(object_detection): report per-frame time, not round-trip,
 for async detectors

AsyncDetectorRunner derived inference_speed from the send_input-to-receive_output
round-trip, which for stream-based detectors (memryx, axelera) is gated by the
SDK pipeline depth and the input cadence (latency ~= depth x input gap): at a
realistic ~35 fps aggregate it reads ~150-200 ms even though the accelerator
handles a frame in ~14 ms back-to-back, making a fast detector look ~10x slower
than sync detectors (which report a single detect_raw duration).

Derive inference_speed from result cadence instead: the rolling average of the
time between completed results equals 1000 / detection_fps, the per-frame
processing time, comparable across detectors. Idle gaps between results are not
folded in, so a quiet stream never inflates the reported speed. The per-send
timestamp deque is removed (it existed only for the old metric).
---
 frigate/object_detection/base.py | 40 ++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/frigate/object_detection/base.py b/frigate/object_detection/base.py
index eca9b72d73..764a5f44e7 100644
--- a/frigate/object_detection/base.py
+++ b/frigate/object_detection/base.py
@@ -4,7 +4,6 @@ import queue
 import threading
 import time
 from abc import ABC, abstractmethod
-from collections import deque
 from multiprocessing import Queue, Value
 from multiprocessing.synchronize import Event as MpEvent
 from typing import Any
@@ -186,6 +185,20 @@ class DetectorRunner(FrigateProcess):
 
 
 class AsyncDetectorRunner(FrigateProcess):
+    # Stream-based detectors (memryx, axelera) keep several frames in flight
+    # through a pipelined SDK, so the send_input -> receive_output round-trip of
+    # one frame is gated by the pipeline depth and the input cadence (measured
+    # latency ~= depth x input gap): at a realistic ~35 fps aggregate that reads
+    # ~150 ms even though the accelerator handles a frame in ~14 ms back-to-back.
+    # Using that as inference_speed makes a fast detector look ~10x slower than
+    # sync detectors, whose metric is the duration of a single detect_raw call.
+    # For async runners, inference_speed is therefore derived from detector
+    # throughput: the average time between completed results equals 1000 /
+    # detection_fps, which is the per-frame processing time and is comparable
+    # across detectors. Idle gaps (> ASYNC_RESULT_IDLE_INTERVAL between results)
+    # are not folded in, so a quiet stream cannot inflate the reported time.
+    ASYNC_RESULT_IDLE_INTERVAL = 0.5
+
     def __init__(
         self,
         name: str,
@@ -208,7 +221,7 @@ class AsyncDetectorRunner(FrigateProcess):
         self._frame_manager: SharedMemoryFrameManager | None = None
         self._publisher: ObjectDetectorPublisher | None = None
         self._detector: AsyncLocalObjectDetector | None = None
-        self.send_times: deque[float] = deque()
+        self._last_result_time: float | None = None
 
     def create_output_shm(self, name: str) -> None:
         out_shm = UntrackedSharedMemory(name=f"out-{name}", create=False)
@@ -238,8 +251,8 @@ class AsyncDetectorRunner(FrigateProcess):
                 logger.warning(f"Failed to get frame {connection_id} from SHM")
                 continue
 
-            # mark start time and send to accelerator
-            self.send_times.append(time.perf_counter())
+            # send to accelerator (no per-send timestamp: inference speed is
+            # derived from result cadence, see class docstring)
             assert self._detector is not None
             self._detector.async_send_input(input_frame, connection_id)
 
@@ -253,12 +266,6 @@ class AsyncDetectorRunner(FrigateProcess):
             if connection_id is None:
                 continue
 
-            if not self.send_times:
-                # guard; shouldn't happen if send/recv are balanced
-                continue
-            ts = self.send_times.popleft()
-            duration = time.perf_counter() - ts
-
             # release input buffer
             assert self._frame_manager is not None
             self._frame_manager.close(connection_id)
@@ -272,8 +279,17 @@ class AsyncDetectorRunner(FrigateProcess):
             assert self._publisher is not None
             self._publisher.publish(connection_id)
 
-            # update timers
-            self.avg_speed.value = (self.avg_speed.value * 9 + duration) / 10
+            # update timers: per-frame processing time derived from result
+            # cadence (1000 / detection_fps). Idle gaps are not folded in, so
+            # a quiet stream never inflates the reported inference speed.
+            now = time.perf_counter()
+            if self._last_result_time is not None:
+                interval = now - self._last_result_time
+                if interval < self.ASYNC_RESULT_IDLE_INTERVAL:
+                    self.avg_speed.value = (
+                        self.avg_speed.value * 9 + interval
+                    ) / 10
+            self._last_result_time = now
             self.start_time.value = 0.0
 
     def run(self) -> None: