From 4e2e6bdc286e627d41a509bcc60559a03749a371 Mon Sep 17 00:00:00 2001 From: patrykorwat Date: Thu, 3 Sep 2026 11:40:05 +0000 Subject: [PATCH] axelera: review round 2 (downloader, stop release, pixel format, docs size, per-class NMS) - download model zips through ModelDownloader (atomic .part rename; a truncated download no longer poisons the cache) - stop(): release the Context (releases connection/model/instance); the previous instance.free() call never existed in the runtime API and was silently swallowed - honor input_pixel_format: flip to RGB planes only when Frigate delivers BGR (input_pixel_format: bgr); with rgb the frame passes through as-is - docs/catalog: width/height = compiled 640x640 input (square regions no longer get squashed twice), document input_pixel_format: bgr - NMS via xyxy_to_xywh_for_nms + per-class cv2.dnn.NMSBoxes, matching the other detector plugins instead of a hand-rolled class-agnostic pass --- docs/data/object_detectors_models.yaml | 11 +-- docs/docs/configuration/object_detectors.md | 5 +- frigate/detectors/plugins/axelera.py | 88 ++++++++++++--------- 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/docs/data/object_detectors_models.yaml b/docs/data/object_detectors_models.yaml index 2fcbe652b7..6653285b2e 100644 --- a/docs/data/object_detectors_models.yaml +++ b/docs/data/object_detectors_models.yaml @@ -1220,9 +1220,9 @@ axelera: | ---------------------------------------- | ------------------------------------------------------------ | | **Custom object detector model path** | `/config/model_cache/axelera/yolox-s-coco-onnx` | | **Label map for custom object detector** | `/labelmap/coco-80.txt` | - | **Object detection model input width** | `704` (the frame size Frigate sends; boxes map back to it) | - | **Object detection model input height** | `576` | - | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Object detection model input width** | `640` (the compiled input size) | + | **Object detection model input height** | `640` | + | **Model Input Pixel Color Format** | `bgr` (Frigate delivers BGR; the plugin hands RGB planes to the compiled input) | | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | | **Model Input D Type** | `int` (Frigate's default value) | | **Object Detection Model Type** | `yolox` | @@ -1232,6 +1232,7 @@ axelera: - axelera:PCIe path: /config/model_cache/axelera/yolox-s-coco-onnx model_type: yolox - width: 704 - height: 576 + width: 640 + height: 640 + input_pixel_format: bgr labelmap_path: /labelmap/coco-80.txt diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index bcb3596a02..8da1e00baa 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -890,8 +890,9 @@ models: - devices: - axelera:PCIe path: /config/model_cache/axelera/yolox-s-coco-onnx - width: 704 # the frame size Frigate sends to the model - height: 576 + width: 640 # the compiled input size; Frigate resizes square regions to it + height: 640 + input_pixel_format: bgr # Frigate delivers BGR; the plugin feeds RGB planes model_type: yolox labelmap_path: /labelmap/coco-80.txt ``` diff --git a/frigate/detectors/plugins/axelera.py b/frigate/detectors/plugins/axelera.py index 62881a7260..92aaf00a8b 100644 --- a/frigate/detectors/plugins/axelera.py +++ b/frigate/detectors/plugins/axelera.py @@ -37,7 +37,6 @@ import os import queue import re import threading -import urllib.request import zipfile from typing import ClassVar, Literal @@ -48,6 +47,7 @@ from pydantic import ConfigDict, Field from frigate.const import MODEL_CACHE_DIR from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum +from frigate.util.model import xyxy_to_xywh_for_nms from frigate.util.runtime_deps import Artifact, ArtifactKind, RuntimeManifest logger = logging.getLogger(__name__) @@ -218,7 +218,9 @@ def _resolve_model_json(model_path, model_url, build_root) -> str: os.makedirs(cache, exist_ok=True) logger.info("axelera: downloading model from %s", model_url) try: - urllib.request.urlretrieve(_url_with_scheme(model_url), zip_path) + from frigate.util.downloader import ModelDownloader + + ModelDownloader.download_from_url(_url_with_scheme(model_url), zip_path) except Exception as exc: raise RuntimeError( f"Failed to download model from {model_url}: {exc}" @@ -284,39 +286,40 @@ def _find_model_json(where: str) -> str: ) -def _nms_xyxy(boxes, scores, iou_threshold: float) -> np.ndarray: - """Greedy NMS on xyxy boxes via cv2.dnn.NMSBoxes (C++, GIL-free). +def _nms_per_class(boxes, scores, class_ids, iou_threshold: float) -> np.ndarray: + """Per-class greedy NMS via cv2.dnn.NMSBoxes (C++, GIL-free). - Returns kept indices in descending score order, same contract as the - previous pure-Python loop but without the O(n^2) worst case on dense - frames (daytime, multiple motion regions at 48 fps offered). + Boxes are xyxy; xyxy_to_xywh_for_nms converts them to the top-left+size + format NMSBoxes expects. Running NMS per class keeps a confident car from + suppressing an overlapping pedestrian, which class-agnostic NMS does. + + Returns kept indices sorted by descending score. """ if len(boxes) == 0: return np.array([], dtype=np.int64) - x1 = boxes[:, 0] - y1 = boxes[:, 1] - xywh = np.stack( - [ - x1, - y1, - np.maximum(0, boxes[:, 2] - x1), - np.maximum(0, boxes[:, 3] - y1), - ], - axis=1, - ) - indices = cv2.dnn.NMSBoxes( - xywh.tolist(), - [float(s) for s in scores], - # inputs are already filtered above CONF_THRESHOLD by the caller - score_threshold=0.0, - nms_threshold=float(iou_threshold), - ) - if len(indices) == 0: + + order = np.argsort(-scores) + kept: list[int] = [] + for cls in np.unique(class_ids): + idx = order[class_ids[order] == cls] + if len(idx) == 0: + continue + indices = cv2.dnn.NMSBoxes( + xyxy_to_xywh_for_nms(boxes[idx]).tolist(), + [float(s) for s in scores[idx]], + # inputs are already filtered above CONF_THRESHOLD by the caller + score_threshold=0.0, + nms_threshold=float(iou_threshold), + ) + if isinstance(indices, np.ndarray): + kept.extend(idx[indices.reshape(-1)].tolist()) + elif len(indices) > 0: + kept.extend(idx[np.asarray(indices, dtype=np.int64).reshape(-1)].tolist()) + + if not kept: return np.array([], dtype=np.int64) - if isinstance(indices, np.ndarray): - return indices.reshape(-1).astype(np.int64) - # OpenCV versions that return a (N, 1) list of nested lists - return np.asarray(indices, dtype=np.int64).reshape(-1) + kept_arr = np.asarray(kept, dtype=np.int64) + return kept_arr[np.argsort(-scores[kept_arr])] def _read_model_manifest(model_dir: str) -> dict: @@ -389,12 +392,16 @@ class _AxeleraRuntimeInference: labelmap: dict, width: int, height: int, + pixel_format: str = "bgr", runtime_factory=None, onnx_factory=None, ) -> None: self._labels = labels self._labelmap = labelmap self._width, self._height = int(width), int(height) + # the compiled model consumes RGB planes; Frigate delivers BGR when + # input_pixel_format is bgr (flip below), RGB when rgb (no flip) + self._flip_channels = pixel_format == "bgr" self._frame_q: queue.Queue = queue.Queue(maxsize=QUEUE_MAXSIZE) self._result_q: queue.Queue = queue.Queue(maxsize=QUEUE_MAXSIZE) self._stop = threading.Event() @@ -585,7 +592,10 @@ class _AxeleraRuntimeInference: if t is not None and t.is_alive(): t.join(timeout=2) try: - self._instance.free() + # Context.release() releases the context and every object it + # created (connection, model, instance); there is no per-object + # free() in this runtime API + self._ctx.release() except Exception: # noqa: BLE001, S110 pass logger.info("axelera: runtime inference stopped") @@ -655,9 +665,12 @@ class _AxeleraRuntimeInference: y0, y1 = self._pad_t, self._pad_t + self._new_h x0, x1 = self._pad_l, self._pad_l + self._new_w ct, cl = self._content_top, self._content_left - # channels 0..2 = quantized RGB (BGR -> RGB flip as in the SDK examples; - # the compiled input expects RGB semantics), channel 3 = alpha-255 - buf[0, ct + y0 : ct + y1, cl + x0 : cl + x1, 0:3] = q[..., ::-1] + # channels 0..2 = quantized RGB: Frigate delivers BGR under the + # documented input_pixel_format: bgr, so flip to the RGB semantics the + # compiled input expects; with input_pixel_format: rgb the frame + # already carries RGB and must pass through untouched + src = q[..., ::-1] if self._flip_channels else q + buf[0, ct + y0 : ct + y1, cl + x0 : cl + x1, 0:3] = src buf[0, ct + y0 : ct + y1, cl + x0 : cl + x1, 3] = 127 def _decode(self, out_bufs) -> np.ndarray: @@ -708,8 +721,8 @@ class _AxeleraRuntimeInference: x1i = np.clip(bx[:, 1], 0, self._width) y0i = np.clip(by[:, 0], 0, self._height) y1i = np.clip(by[:, 1], 0, self._height) - # NMS on the model-space boxes (scale-invariant IoU) - keep_nms = _nms_xyxy(boxes, conf, NMS_IOU_THRESHOLD) + # NMS on the model-space boxes (scale-invariant IoU), per class + keep_nms = _nms_per_class(boxes, conf, cls_idx, NMS_IOU_THRESHOLD) rows = [] for i in keep_nms: label = ( @@ -802,6 +815,9 @@ class AxeleraDetector(DetectionApi): labelmap=self._labelmap, width=self.width, height=self.height, + pixel_format=str( + getattr(model.input_pixel_format, "value", model.input_pixel_format) + ), ) logger.info( "axelera: runtime model=%s (%dx%d input, frame %dx%d) ready",