From 231c648e87e874282143c4d1143ec9a1e0c792a1 Mon Sep 17 00:00:00 2001 From: Mika <19797875+it-rec@users.noreply.github.com> Date: Wed, 5 Aug 2026 04:56:57 +0000 Subject: [PATCH 1/2] Optimize per-frame CPU and memory in the video pipeline - Replace the full-frame float64 clip-and-rescale in motion contrast enhancement with a 256-entry uint8 LUT applied via cv2.LUT. This removes several full-frame float64 temporaries per frame while producing a bit-identical result (verified), and merges the two np.percentile calls into one partition. - Convert the frame to BGR once per match_and_update instead of once per detection for autotracking histogram embeddings, by splitting get_histogram_from_bgr out of get_histogram. Output is unchanged. - Read ffmpeg frames directly into shared memory with stdout.readinto instead of read() + slice assignment, removing a full-frame allocation and copy per frame in the capture thread. - Hoist the invariant /proc/uptime read and the SC_CLK_TCK / SC_PAGE_SIZE sysconf lookups out of the per-process loop in get_cpu_stats. --- frigate/motion/improved_motion.py | 18 +++++++++++++----- frigate/track/norfair_tracker.py | 16 ++++++++++++---- frigate/util/image.py | 8 ++++++++ frigate/util/services.py | 17 ++++++++++------- frigate/video/ffmpeg.py | 10 +++++++++- 5 files changed, 52 insertions(+), 17 deletions(-) diff --git a/frigate/motion/improved_motion.py b/frigate/motion/improved_motion.py index 598eeacd1d..a15922c34d 100644 --- a/frigate/motion/improved_motion.py +++ b/frigate/motion/improved_motion.py @@ -86,8 +86,9 @@ class ImprovedMotionDetector(MotionDetector): # Improve contrast if self.config.improve_contrast: # TODO tracking moving average of min/max to avoid sudden contrast changes - min_value = np.percentile(resized_frame, 4).astype(np.uint8) - max_value = np.percentile(resized_frame, 96).astype(np.uint8) + min_value, max_value = np.percentile(resized_frame, [4, 96]).astype( + np.uint8 + ) # skip contrast calcs if the image is a single color if min_value < max_value: # keep track of the last 50 contrast values @@ -101,10 +102,17 @@ class ImprovedMotionDetector(MotionDetector): avg_min, avg_max = np.mean(self.contrast_values, axis=0) - resized_frame = np.clip(resized_frame, avg_min, avg_max) - resized_frame = ( - ((resized_frame - avg_min) / (avg_max - avg_min)) * 255 + # Apply the clip-and-rescale as a 256-entry uint8 lookup table + # instead of promoting the whole frame to float64 (which + # allocated several full-frame temporaries per frame). The + # per-pixel result is identical since the input is uint8. + contrast_lut = np.clip( + np.arange(256, dtype=np.float64), avg_min, avg_max + ) + contrast_lut = ( + ((contrast_lut - avg_min) / (avg_max - avg_min)) * 255 ).astype(np.uint8) + resized_frame = cv2.LUT(resized_frame, contrast_lut) if self.save_images: contrasted_saved = resized_frame.copy() diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index 6b39885c0d..6158076751 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -25,7 +25,7 @@ from frigate.track.stationary_classifier import ( ) from frigate.util.image import ( SharedMemoryFrameManager, - get_histogram, + get_histogram_from_bgr, intersection_over_union, ) from frigate.util.object import average_boxes, median_of_boxes @@ -523,6 +523,14 @@ class NorfairTracker(ObjectTracker): yuv_frame = self.frame_manager.get( frame_name, self.camera_config.frame_shape_yuv ) + + # Convert the frame to BGR once per call for histogram embeddings rather + # than once per detection; every detection histograms a region of the + # same frame. + bgr_frame: np.ndarray | None = None + if yuv_frame is not None and self.ptz_metrics.autotracker_enabled.value: + bgr_frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420) + for obj in detections: label = obj[0] if label not in detections_by_type: @@ -536,9 +544,9 @@ class NorfairTracker(ObjectTracker): points = np.array([[obj[2][0], obj[2][1]], [obj[2][2], obj[2][3]]]) embedding = None - if self.ptz_metrics.autotracker_enabled.value: - embedding = get_histogram( - yuv_frame, obj[2][0], obj[2][1], obj[2][2], obj[2][3] + if bgr_frame is not None: + embedding = get_histogram_from_bgr( + bgr_frame, obj[2][0], obj[2][1], obj[2][2], obj[2][3] ) detection = Detection( diff --git a/frigate/util/image.py b/frigate/util/image.py index dde25e2425..f4603e2a07 100644 --- a/frigate/util/image.py +++ b/frigate/util/image.py @@ -1251,6 +1251,14 @@ def get_image_from_recording( def get_histogram(image, x_min, y_min, x_max, y_max): image_bgr = cv2.cvtColor(image, cv2.COLOR_YUV2BGR_I420) + return get_histogram_from_bgr(image_bgr, x_min, y_min, x_max, y_max) + + +def get_histogram_from_bgr(image_bgr, x_min, y_min, x_max, y_max): + """Compute a normalized BGR histogram for a region of an already-converted + BGR frame. Callers that need histograms for several regions of the same + frame should convert the frame to BGR once and reuse it here rather than + paying a full-frame YUV to BGR conversion per region.""" image_bgr = image_bgr[y_min:y_max, x_min:x_max] hist = cv2.calcHist( diff --git a/frigate/util/services.py b/frigate/util/services.py index 38737907ef..010da889fe 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -106,7 +106,15 @@ def get_cpu_stats() -> dict[str, dict]: """Get cpu usages for each process id""" usages = {} docker_memlimit = get_docker_memlimit_bytes() / 1024 - total_mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") / 1024 + page_size = os.sysconf("SC_PAGE_SIZE") + total_mem = page_size * os.sysconf("SC_PHYS_PAGES") / 1024 + + # These values are invariant across the process loop below, so read them + # once per stats cycle instead of once per process. On hosts with many + # camera ffmpeg instances the per-process reads were a visible spike. + clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"]) + with open("/proc/uptime") as f: + system_uptime_sec = int(float(f.read().split()[0])) system_cpu = psutil.cpu_percent( interval=None @@ -133,11 +141,6 @@ def get_cpu_stats() -> dict[str, dict]: stime = int(stats[14]) start_time = int(stats[21]) - with open("/proc/uptime") as f: - system_uptime_sec = int(float(f.read().split()[0])) - - clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"]) - process_utime_sec = utime // clk_tck process_stime_sec = stime // clk_tck process_start_time_sec = start_time // clk_tck @@ -148,7 +151,7 @@ def get_cpu_stats() -> dict[str, dict]: with open(f"/proc/{pid}/statm") as f: mem_stats = f.readline().split() - mem_res = int(mem_stats[1]) * os.sysconf("SC_PAGE_SIZE") / 1024 + mem_res = int(mem_stats[1]) * page_size / 1024 if docker_memlimit > 0: mem_pct = round((mem_res / docker_memlimit) * 100, 1) diff --git a/frigate/video/ffmpeg.py b/frigate/video/ffmpeg.py index 24b7805333..5bf8749f55 100644 --- a/frigate/video/ffmpeg.py +++ b/frigate/video/ffmpeg.py @@ -75,7 +75,15 @@ def capture_frames( frame_name = f"{config.name}_frame{frame_index}" frame_buffer = frame_manager.write(frame_name) try: - frame_buffer[:] = ffmpeg_process.stdout.read(frame_size) + # read directly into shared memory to avoid allocating and then + # copying a full frame out of the ffmpeg pipe every frame. stdout + # is buffered, so readinto returns a full frame unless the pipe + # closed mid-frame. + if ( + ffmpeg_process.stdout.readinto(frame_buffer[:frame_size]) + != frame_size + ): + raise EOFError("did not read a full frame from ffmpeg") except Exception: # shutdown has been initiated if stop_event.is_set(): From d93f296691bba288e4e671c6c4cf13244d94d8d0 Mon Sep 17 00:00:00 2001 From: Mika <19797875+it-rec@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:13:56 +0200 Subject: [PATCH 2/2] Skip the BGR conversion when there are no detections Hoisting the YUV to BGR conversion out of the detection loop made it run even on frames with an empty detection list, where the previous per-detection code converted nothing at all. match_and_update is called with an empty list whenever detection is disabled, and whenever motion triggered detection but the detector returned no trackable objects, so this is the common case on a camera that is mostly idle. A wasted full-frame conversion costs 0.69 ms / 2.8 MB at 1280x720 and 2.89 ms / 12.3 MB at 2688x1520 - 4x to 17x more than the contrast LUT saves per frame - which would have made this change a net regression for autotracking cameras. --- frigate/track/norfair_tracker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index 6158076751..986f056ada 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -526,9 +526,14 @@ class NorfairTracker(ObjectTracker): # Convert the frame to BGR once per call for histogram embeddings rather # than once per detection; every detection histograms a region of the - # same frame. + # same frame. Skipped when there are no detections, so a frame without + # objects costs no conversion at all. bgr_frame: np.ndarray | None = None - if yuv_frame is not None and self.ptz_metrics.autotracker_enabled.value: + if ( + detections + and yuv_frame is not None + and self.ptz_metrics.autotracker_enabled.value + ): bgr_frame = cv2.cvtColor(yuv_frame, cv2.COLOR_YUV2BGR_I420) for obj in detections: