From 13883054392b17dee2a39eeb51e234e736a91a84 Mon Sep 17 00:00:00 2001 From: aahmetdurmaz Date: Thu, 10 Sep 2026 20:09:48 +0300 Subject: [PATCH] Extend PPU layout handling and YOLOX support to DEEPX detector --- docs/data/object_detectors_models.yaml | 54 ++- docs/docs/configuration/object_detectors.md | 9 +- docs/docs/frigate/hardware.md | 6 +- frigate/detectors/plugins/deepx.py | 386 +++++++++++++++--- frigate/test/test_deepx.py | 421 +++++++++++++++++++- 5 files changed, 796 insertions(+), 80 deletions(-) diff --git a/docs/data/object_detectors_models.yaml b/docs/data/object_detectors_models.yaml index adf5a51bdc..909941b7e1 100644 --- a/docs/data/object_detectors_models.yaml +++ b/docs/data/object_detectors_models.yaml @@ -991,31 +991,59 @@ deepx: - key: yolo label: YOLO recommended: true - download: No model is bundled with Frigate. Download a pre-compiled YOLO `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point the model's `path` at it. The output layout is read from the compiled model, so anchor-based, anchor-free, NMS-in-head and PPU models (anchor-based or anchor-free) all need no extra configuration. + download: No model is bundled with Frigate. Download a pre-compiled YOLO `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point the model's `path` at it. The recommended model is `yolox-s_640x640_ppu.dxnn`. Its Post-Processing Unit (PPU) compile moves candidate selection onto the NPU, which makes it the fastest ModelZoo model measured through Frigate (about 13 ms on a DX-M1). The output layout is read from the compiled model, so anchor-based, anchor-free, NMS-in-head and PPU models (anchor-based or anchor-free) all need no extra configuration; prefer a `PPU` variant whenever the ModelZoo offers one. ui: |- Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure: - | Field | Value | - | ---------------------------------------- | -------------------------------------- | - | **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` | - | **Label map for custom object detector** | `/labelmap/coco-80.txt` | - | **Object detection model input width** | `640` | - | **Object detection model input height** | `640` | - | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | - | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | - | **Model Input D Type** | `int` (Frigate's default value) | - | **Object Detection Model Type** | `yolo-generic` | + | Field | Value | + | ---------------------------------------- | ---------------------------------------------------- | + | **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640_ppu.dxnn` | + | **Label map for custom object detector** | `/labelmap/coco-80.txt` | + | **Object detection model input width** | `640` | + | **Object detection model input height** | `640` | + | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | + | **Model Input D Type** | `int` (Frigate's default value) | + | **Object Detection Model Type** | `yolo-generic` | - The width and height must match the resolution the `.dxnn` file was compiled for. Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden. + Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden. yaml: |- models: - devices: - deepx:PCIe:0 - path: /config/model_cache/deepx/model.dxnn + path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn labelmap_path: /labelmap/coco-80.txt model_type: yolo-generic width: 640 height: 640 + - key: yolox + label: YOLOX + recommended: false + download: No model is bundled with Frigate. Download a pre-compiled YOLOX `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point the model's `path` at it. The `_ppu` variant is faster and also works with the `yolo-generic` model type; the plain export needs `yolox` so its raw head is decoded. + ui: |- + Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure: + + | Field | Value | + | ---------------------------------------- | ---------------------------------------- | + | **Custom object detector model path** | `/config/model_cache/deepx/yolox-s_640x640.dxnn`| + | **Label map for custom object detector** | `/labelmap/coco-80.txt` | + | **Object detection model input width** | `640` | + | **Object detection model input height** | `640` | + | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | + | **Model Input D Type** | `int` (Frigate's default value) | + | **Object Detection Model Type** | `yolox` | + + The width and height must match the resolution the `.dxnn` file was compiled for. + yaml: |- + models: + - devices: + - deepx:PCIe:0 + path: /config/model_cache/deepx/yolox-s_640x640.dxnn + labelmap_path: /labelmap/coco-80.txt + model_type: yolox + width: 640 + height: 640 - key: damoyolo label: DAMO-YOLO (TinyNAS-L20T / S / M / L) recommended: false diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 0fc16e8610..6392e50c2c 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -655,25 +655,26 @@ The DX-RT Python bindings are not part of the Frigate image. They are downloaded -Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. Two model types are supported: +Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. Three model types are supported: - `yolo-generic` for YOLO object detection models, the recommended default. The detector reads the model's output layout from the compiled file, so anchor-based, anchor-free and NMS-in-head models all work with the same configuration, as do models compiled with DEEPX's Post-Processing Unit (PPU) support. +- `yolox` for YOLOX models compiled without PPU support, whose raw head needs Frigate's YOLOX decoder. A YOLOX model compiled with PPU support works under either `yolox` or `yolo-generic`. - `damo-yolo` for DAMO-YOLO models. All four sizes (TinyNAS-L20T, S, M, L) are supported. -The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of YOLO and DAMO-YOLO object detection models. Download the `.dxnn`, bind-mount it into the container, and point the model's `path` at it. Alternatively, compile your own model with the DX-COM compiler. +The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of YOLO and DAMO-YOLO object detection models. Download the `.dxnn`, bind-mount it into the container, and point the model's `path` at it. Alternatively, compile your own model with the DX-COM compiler. The recommended starting point is `yolox-s_640x640_ppu.dxnn`, the fastest ModelZoo model measured through Frigate: ```yaml models: - devices: - deepx:PCIe:0 - path: /config/model_cache/deepx/model.dxnn + path: /config/model_cache/deepx/yolox-s_640x640_ppu.dxnn labelmap_path: /labelmap/coco-80.txt model_type: yolo-generic width: 640 height: 640 ``` -`model_type` must be set to `yolo-generic` or `damo-yolo` to match the model; `yolo-generic` is the recommended default unless the model is specifically a DAMO-YOLO export. Frigate defaults it to `ssd`, which this detector does not support, so a model that leaves it unset is rejected at startup. +`model_type` must be set to `yolo-generic`, `yolox` or `damo-yolo` to match the model; `yolo-generic` is the recommended default unless the model is a raw YOLOX or DAMO-YOLO export. Frigate defaults it to `ssd`, which this detector does not support, so a model that leaves it unset is rejected at startup. `width` and `height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden. diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 9b02164848..2b2889eb17 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -267,9 +267,9 @@ Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 mo DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com). -The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo). Models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage; both anchor-based and anchor-free PPU heads are supported. +The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo). Models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage; both anchor-based and anchor-free PPU heads are supported. Frigate reads the PPU head's kind and number of detection scales from the `.dxnn` file when the model loads, so no configuration is needed for either. -Exception: **YOLOv7 PPU models are not supported**. A PPU record has no anchor sizes, only a grid position, so Frigate decodes anchor-based PPU heads against a fixed anchor table; YOLOv7's anchors differ, so it decodes to wrong box sizes. Non-PPU YOLOv7 is unaffected (its raw head already outputs pixel-space boxes). +Exception: **YOLOv7 and YOLOv7-x PPU models are not supported**. A PPU record has no anchor sizes, only a grid position, and the `.dxnn` does not carry them either, so Frigate decodes anchor-based PPU heads against a fixed anchor table per scale count: the usual COCO anchor set for three-scale heads (the one DEEPX's own reference decoder uses for every other three-scale model it ships) and the usual tiny set for two-scale heads. YOLOv7's anchors differ and nothing in the model tells them apart, so it decodes to wrong box sizes. Non-PPU YOLOv7 is unaffected (its raw head already outputs pixel-space boxes). An end-to-end PPU head that emits corner boxes rather than a centre and size is told apart from the records themselves. **Face and pose PPU models (SCRFD and the pose export) are not supported**: their records carry landmarks or keypoints in a different layout, and Frigate only decodes object detection records. The DEEPX kernel driver, the DX-RT runtime, and the `dxrtd` daemon all run on the Docker host rather than inside the Frigate container, and have to be installed there before the NPU can be used. Frigate connects to the daemon over its socket, so the NPU stays available to other programs on the host at the same time. See the [installation docs](installation.md#deepx-npu) for the setup steps. @@ -287,7 +287,7 @@ The DEEPX ModelZoo publishes pre-compiled `.dxnn` files for many YOLO variants a These are DEEPX's own figures for the DX-M1, derived from the frames per second published in the ModelZoo. They cover the NPU alone and exclude the pre- and post-processing Frigate does on the host, so the inference speed Frigate reports will be higher, and noticeably so on a slower host such as a Raspberry Pi 5. -Besides the DAMO-YOLO family shown above, other models from the wider YOLO family published in the ModelZoo are also supported by the `deepx` detector by setting `model_type: yolo-generic`. Inference times for these, measured through Frigate's own stats API on a DX-M1 rather than taken from DEEPX's published figures, are below: +Besides the DAMO-YOLO family shown above, other models from the wider YOLO family published in the ModelZoo are also supported by the `deepx` detector by setting `model_type: yolo-generic`. A YOLOX model compiled without PPU support is the one exception: its raw head needs `model_type: yolox` (the PPU variant works with either). Inference times for these, measured through Frigate's own stats API on a DX-M1 rather than taken from DEEPX's published figures, are below: | Model | Input Size | DX-M1 Inference Time (via Frigate) | | ------------------ | ---------- | ----------------------------------- | diff --git a/frigate/detectors/plugins/deepx.py b/frigate/detectors/plugins/deepx.py index 6e98b704c1..538e63cb0c 100644 --- a/frigate/detectors/plugins/deepx.py +++ b/frigate/detectors/plugins/deepx.py @@ -1,7 +1,10 @@ """DEEPX NPU detector running compiled .dxnn models via the DX-RT runtime.""" +import json import logging import os +import re +import struct from dataclasses import dataclass from enum import Enum from typing import Literal @@ -12,7 +15,7 @@ from pydantic import ConfigDict, Field, field_validator, model_validator from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum -from frigate.util.model import post_process_yolo +from frigate.util.model import post_process_yolo, post_process_yolox from frigate.util.runtime_deps import Artifact, ArtifactKind, RuntimeManifest logger = logging.getLogger(__name__) @@ -33,7 +36,14 @@ DXRT_IPC_SOCKET = "/run/dxrt/dxrt_dynamic_ipc.sock" SCORE_THRESHOLD = 0.4 NMS_THRESHOLD = 0.4 -SUPPORTED_MODEL_TYPES = (ModelTypeEnum.yologeneric, ModelTypeEnum.damoyolo) +SUPPORTED_MODEL_TYPES = ( + ModelTypeEnum.yologeneric, + ModelTypeEnum.yolox, + ModelTypeEnum.damoyolo, +) + +# YOLOX's raw head concatenates one cell per position of its three strides +YOLOX_STRIDES = (8, 16, 32) # Fixed-width record the PPU emits, DeviceBoundingBox_t in DX-RT's # datatype.h: x, y, w, h (float32), grid_y, grid_x, box_idx, layer_idx @@ -44,10 +54,14 @@ PPU_GRID_BYTES = (16, 20) PPU_SCORE_BYTES = (20, 24) PPU_LABEL_BYTES = (24, 28) +# Anchor sizes are not in the PPU record or the .dxnn, so an anchor-based +# head is read against the table for its scale count: DEEPX's own reference +# decoder's tables, shared by every three- or two-scale model it ships +# except YOLOv7, whose anchors differ. PPU_ANCHORS_BY_SCALES = { 2: np.array( [ - [[23, 27], [37, 58], [81, 82]], + [[10, 14], [23, 27], [37, 58]], [[81, 82], [135, 169], [344, 319]], ], dtype=np.float32, @@ -68,6 +82,18 @@ PPU_MAX_STRIDE = 32 # gets the biggest table as a best guess PPU_MAX_KNOWN_SCALES = max(PPU_ANCHORS_BY_SCALES) +# .dxnn: "DXNN", a 4-byte LE version, then a JSON index padded to +# DXNN_HEADER_SIZE; every offset in the index counts from there. +DXNN_MAGIC = b"DXNN" +DXNN_HEADER_SIZE = 8192 +# compile_config's ppu.type for the heads decoded as bounding boxes +PPU_TYPE_ANCHOR_BASED = 0 +PPU_TYPE_ANCHOR_FREE = 1 +# The PPU tensor table, ppu_info_header_t then one ppu_info_t per output +# tensor, as DX-RT's ppu_binary_parser.h defines them. +PPU_TABLE_HEADER = struct.Struct(" int: + """Fail unless the shapes are YOLOX's raw (1, N, 5+C) head for this + input size; returns the row width to orient a channel-major export.""" + columns = 5 + num_classes + cells = sum((height // s) * (width // s) for s in YOLOX_STRIDES) + dims = [_significant_dims(shape) for shape in shapes] + + if ( + len(dims) == 1 + and len(dims[0]) == 2 + and sorted(dims[0]) == sorted((cells, columns)) + ): + return columns + + raise ValueError( + f"output shapes {[tuple(shape) for shape in shapes]} are not YOLOX's " + f"(1, {cells}, {columns}) raw head for a {width}x{height} input. Check " + "that width and height match the compiled model, that labelmap_path " + "matches it, usually /labelmap/coco-80.txt, and that model_type " + "matches it; any other YOLO head needs model_type yolo-generic." + ) + + def rows_with_columns(tensor: np.ndarray, columns: int) -> np.ndarray: """(N, columns) rows, transposing a channel-major export first.""" if tensor.ndim >= 2 and tensor.shape[-1] != columns and tensor.shape[-2] == columns: @@ -408,17 +460,11 @@ def ppu_boxes_are_anchor_based(boxes: np.ndarray) -> bool | None: return None -def ppu_layout_is_anchor_based(outputs: list[np.ndarray]) -> bool | None: - """Whether this PPU model's records are anchor-based, or None when the - frame cannot tell; the caller keeps the first conclusive answer.""" - records = ppu_records(outputs) - - if records is None: - return None - - boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) - - return ppu_boxes_are_anchor_based(boxes) +def ppu_boxes_are_centres(boxes: np.ndarray) -> bool: + """Whether these pixel boxes hold a centre and size rather than two + corners: a corner box always runs x1<=x2, y1<=y2, so only a centre box + can show a width below its x or a height below its y.""" + return bool(np.any(boxes[:, 2] < boxes[:, 0]) or np.any(boxes[:, 3] < boxes[:, 1])) def ppu_needs_grid_decode(records: np.ndarray) -> bool: @@ -429,6 +475,134 @@ def ppu_needs_grid_decode(records: np.ndarray) -> bool: return len(np.unique(layer_idx)) > 1 +@dataclass(frozen=True) +class PpuLayout: + """The PPU head's kind (None if unnamed) and each scale's grid, finest + first, as the compiled model states them.""" + + anchor_based: bool | None + grids: tuple[tuple[int, int], ...] + + @property + def scale_count(self) -> int: + return len(self.grids) + + +def read_ppu_layout(path: str) -> PpuLayout | None: + """Read the PPU head layout the compiler wrote into a .dxnn, or None + when there is none to read; only the index and metadata are read.""" + try: + with open(path, "rb") as model: + header = model.read(DXNN_HEADER_SIZE) + if header[:4] != DXNN_MAGIC: + return None + + index = json.JSONDecoder().raw_decode( + header[8:].decode("utf-8", "replace") + )[0] + data = index["data"] + + def section(entry: dict) -> bytes: + model.seek(DXNN_HEADER_SIZE + int(entry["offset"])) + return model.read(int(entry["size"])) + + anchor_based = None + ppu = None + if data.get("compile_config"): + ppu = json.loads(section(data["compile_config"])).get("ppu") + if ppu is None: + return None + + anchor_based = { + PPU_TYPE_ANCHOR_BASED: True, + PPU_TYPE_ANCHOR_FREE: False, + }.get(ppu.get("type")) + + for chip in data.get("compiled_data", {}).values(): + for npu in chip.values(): + # v8 has a PPU table; v7's output tensor shapes hold the same grids + if npu.get("ppu", {}).get("size"): + grids = _ppu_grids_from_table(section(npu["ppu"])) + elif npu.get("rmap_info", {}).get("size"): + outputs = json.loads(section(npu["rmap_info"])).get( + "outputs", [] + ) + grids = _ppu_grids_from_outputs(outputs, ppu) + if grids and anchor_based is None: + anchor_based = _ppu_outputs_are_anchor_based(outputs, ppu) + else: + continue + + if grids: + return PpuLayout( + anchor_based, tuple(grids[k] for k in sorted(grids)) + ) + + return None + except (OSError, ValueError, KeyError, TypeError, AttributeError, struct.error): + return None + + +def _ppu_grids_from_table(table: bytes) -> dict[int, tuple[int, int]]: + """Grid per scale index from the PPU table: field 3 is the scale, + fields 8 and 9 its grid, shared by every anchor of that scale.""" + _, tensor_count, _, _ = PPU_TABLE_HEADER.unpack_from(table) + if len(table) < PPU_TABLE_HEADER.size + tensor_count * PPU_TABLE_ENTRY.size: + raise ValueError("PPU table shorter than its tensor count") + + grids: dict[int, tuple[int, int]] = {} + for i in range(tensor_count): + fields = PPU_TABLE_ENTRY.unpack_from( + table, PPU_TABLE_HEADER.size + i * PPU_TABLE_ENTRY.size + ) + grids.setdefault(fields[3], (fields[8], fields[9])) + + return grids + + +def _ppu_output_scale(name: str, ppu: dict | None) -> int | None: + """Which scale a PPU output tensor belongs to, from ppu.outputs by + name or else the name's own PPU_..._Output_[_anchor_] form.""" + entry = ((ppu or {}).get("outputs") or {}).get(name) + if isinstance(entry, dict) and "conv_idx" in entry: + return int(entry["conv_idx"]) + + match = re.fullmatch(r"PPU_\w*?Output_(\d+)(?:_anchor_\d+)?", name) + return int(match.group(1)) if match else None + + +def _ppu_grids_from_outputs( + outputs: list[dict], ppu: dict | None +) -> dict[int, tuple[int, int]]: + """Grid per scale index from rmap_info's output tensor shapes.""" + grids: dict[int, tuple[int, int]] = {} + for output in outputs: + scale = _ppu_output_scale(str(output.get("name", "")), ppu) + shape = output.get("shape") or [] + if scale is None or len(shape) not in (3, 4): + continue + + grid = (int(shape[2]), int(shape[1])) if len(shape) == 4 else (int(shape[1]), 1) + grids.setdefault(scale, grid) + + return grids + + +def _ppu_outputs_are_anchor_based(outputs: list[dict], ppu: dict | None) -> bool | None: + """Anchor-based when the PPU outputs are split per anchor; None when + nothing says either way.""" + mapping = (ppu or {}).get("outputs") or {} + if any( + isinstance(entry, dict) and "anchor_idx" in entry for entry in mapping.values() + ): + return True + + if any("_anchor_" in str(output.get("name", "")) for output in outputs): + return True + + return None + + def ppu_scale_count_lower_bound(records: np.ndarray, width: int, height: int) -> int: """The fewest detection scales these records can come from. Each proves one past its layer index, and its grid cell can prove more: cell g of @@ -537,19 +711,21 @@ def decode_ppu( scale_count: int | None = None, needs_grid_decode: bool | None = None, records: np.ndarray | None = None, + centre_boxes: bool | None = None, + boxes: np.ndarray | None = None, ) -> np.ndarray: - """Decode PPU records, leaving only NMS. `anchor_based` and - `needs_grid_decode` skip their heuristics once settled by a caller - tracking them across frames; `scale_count` picks the anchor table or - stride, defaulting to what this frame alone shows; `records` reuses an - already-parsed buffer instead of re-parsing `outputs`.""" + """Decode PPU records, leaving only NMS. `anchor_based`, + `needs_grid_decode` and `centre_boxes` skip their heuristics once a + caller settles them; `records` and `boxes` reuse an already-parsed + buffer instead of re-parsing `outputs`.""" if records is None: records = ppu_records(outputs) if records is None: return np.zeros((20, 6), np.float32) - boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) + if boxes is None: + boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) if anchor_based is None: anchor_based = ppu_boxes_are_anchor_based(boxes) @@ -579,8 +755,17 @@ def decode_ppu( ) scores = np.where(known, scores, 0.0) else: - centre_x, centre_y = boxes[:, 0], boxes[:, 1] - box_w, box_h = boxes[:, 2], boxes[:, 3] + if centre_boxes is None: + centre_boxes = ppu_boxes_are_centres(boxes) + + if centre_boxes: + centre_x, centre_y = boxes[:, 0], boxes[:, 1] + box_w, box_h = boxes[:, 2], boxes[:, 3] + else: + box_w = boxes[:, 2] - boxes[:, 0] + box_h = boxes[:, 3] - boxes[:, 1] + centre_x = boxes[:, 0] + box_w * 0.5 + centre_y = boxes[:, 1] + box_h * 0.5 x_min = centre_x - box_w * 0.5 y_min = centre_y - box_h * 0.5 @@ -836,15 +1021,77 @@ class DeepxDetector(DetectionApi): self.session = InferenceEngine(str(model_path), options) self.output = self.inspect_model(config) + if self.output is not None and self.output.layout is YoloLayout.yolox: + # the shared YOLOX decoder's cell grids and strides for this input + self.calculate_grids_strides() self.logged_layout = False # set once a frame proves the PPU head anchor-free, then kept self.ppu_anchor_free = False # set once proven needed, then kept self.ppu_needs_grid_decode = False self.ppu_scale_count = 0 + # set once proven; a corner-format head never produces such a record + self.ppu_centre_boxes = False self.ppu_unsupported_scale_reported = False self.damoyolo_outputs_reported = False + # the model's own head layout; None falls back to the fields above + self.ppu_layout = None + if self.output is not None and self.output.layout is YoloLayout.ppu: + self.ppu_layout = self.inspect_ppu_head(model_path) + + def inspect_ppu_head(self, model_path: str) -> PpuLayout | None: + """Settle the PPU head from the model file, so the anchor table and + strides are right from the first frame.""" + layout = read_ppu_layout(model_path) + + if layout is None: + logger.warning( + "Could not read the PPU head layout from %s; inferring its " + "scale count from the detections instead, which can misplace " + "boxes until a record proves every scale", + model_path, + ) + return None + + self.ppu_scale_count = layout.scale_count + if layout.anchor_based is not None: + self.ppu_anchor_free = not layout.anchor_based + # only a grid-relative head keeps one tensor per scale + self.ppu_needs_grid_decode = layout.anchor_based is False and ( + layout.scale_count > 1 + ) + + # a head laid out differently than the decoders assume would + # decode to wrong positions, so say so once here + if layout.anchor_based or layout.scale_count > 1: + strides = [ + PPU_MAX_STRIDE >> (layout.scale_count - 1 - i) + for i in range(layout.scale_count) + ] + expected = tuple( + (-(-self.width // s), -(-self.height // s)) for s in strides + ) + if layout.grids != expected: + logger.warning( + "PPU head grids %s do not match the %d-scale layout Frigate " + "decodes (%s for %dx%d input); boxes may be misplaced", + layout.grids, + layout.scale_count, + expected, + self.width, + self.height, + ) + + logger.info( + "DEEPX PPU head from the model: %s, %d scale(s)", + {True: "anchor-based", False: "anchor-free"}.get( + layout.anchor_based, "unknown kind" + ), + layout.scale_count, + ) + return layout + def inspect_model(self, config: DeepxDetectorConfig) -> YoloOutput | None: """Pick the decoder from what the runtime reports, failing here rather than on the first frame.""" @@ -864,19 +1111,27 @@ class DeepxDetector(DetectionApi): validate_damoyolo_outputs(shapes, num_classes) return None - output = infer_yolo_layout( - shapes, - num_classes, - self.session.is_ppu(), - self.session.has_dynamic_output(), - ) + if self.model_type == ModelTypeEnum.yolox and not self.session.is_ppu(): + # a YOLOX compiled with PPU support is read as PPU below + columns = validate_yolox_outputs( + shapes, num_classes, self.width, self.height + ) + output = YoloOutput(YoloLayout.yolox, columns) + else: + output = infer_yolo_layout( + shapes, + num_classes, + self.session.is_ppu(), + self.session.has_dynamic_output(), + ) except ValueError as err: raise ValueError( f"Cannot decode DEEPX model '{config.model.path}': {err}" ) from None logger.info( - "DEEPX decoding yolo-generic output as %s with %d classes", + "DEEPX decoding %s output as %s with %d classes", + self.model_type.value, output.layout.value, num_classes, ) @@ -890,6 +1145,8 @@ class DeepxDetector(DetectionApi): match self.output.layout: case YoloLayout.ppu: return self.decode_ppu(outputs) + case YoloLayout.yolox: + return self.decode_yolox(outputs) case YoloLayout.anchor: return decode_raw_anchor( outputs, @@ -912,29 +1169,54 @@ class DeepxDetector(DetectionApi): # per-scale feature maps, same thresholds as above return post_process_yolo(outputs, self.width, self.height) + def decode_yolox(self, outputs: list[np.ndarray]) -> np.ndarray: + """Decode YOLOX's raw head with Frigate's shared decoder; it writes + into its input, so a float copy is oriented as (1, N, 5+C) first.""" + rows = rows_with_columns(outputs[0], self.output.columns) + predictions = np.array(rows, dtype=np.float32).reshape(1, -1, rows.shape[-1]) + + return post_process_yolox( + predictions, self.width, self.height, self.grids, self.expanded_strides + ) + def decode_ppu(self, outputs: list[np.ndarray]) -> np.ndarray: - """Decode PPU records, keeping each heuristic once a frame settles - it: anchor-free, needs-grid-decode, and scale count all only - move toward more certainty, never back. The scale count is the - largest lower bound any frame has proven, so a three-scale head - whose frames so far only held its finer layers near the origin - reads against the two-scale table until a record lands far enough - right or down, or on the coarsest layer, to prove the third.""" - if self.ppu_anchor_free: + """Decode PPU records. The head layout read from the model fixes + the kind, scale count and grid decode from the first frame; without + it, each is settled from the records and kept once proven.""" + records = ppu_records(outputs) + boxes = None + if records is not None and len(records): + boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) + + if self.ppu_layout is not None and self.ppu_layout.anchor_based is not None: + anchor_based = self.ppu_layout.anchor_based + elif self.ppu_anchor_free: anchor_based = False else: - anchor_based = ppu_layout_is_anchor_based(outputs) + anchor_based = ( + ppu_boxes_are_anchor_based(boxes) if boxes is not None else None + ) self.ppu_anchor_free = anchor_based is False - records = ppu_records(outputs) - if records is not None and len(records): - self.ppu_scale_count = max( - self.ppu_scale_count, - ppu_scale_count_lower_bound(records, self.width, self.height), - ) + if boxes is not None: + if self.ppu_layout is None: + self.ppu_scale_count = max( + self.ppu_scale_count, + ppu_scale_count_lower_bound(records, self.width, self.height), + ) if not anchor_based and not self.ppu_needs_grid_decode: - self.ppu_needs_grid_decode = ppu_needs_grid_decode(records) + if self.ppu_layout is None: + self.ppu_needs_grid_decode = ppu_needs_grid_decode(records) + else: + self.ppu_needs_grid_decode = self.ppu_layout.scale_count > 1 + + if ( + anchor_based is False + and not self.ppu_needs_grid_decode + and not self.ppu_centre_boxes + ): + self.ppu_centre_boxes = ppu_boxes_are_centres(boxes) if ( anchor_based @@ -951,8 +1233,13 @@ class DeepxDetector(DetectionApi): ", ".join(str(n) for n in sorted(PPU_ANCHORS_BY_SCALES)), ) - # None here means nothing conclusive in this frame, and no box to - # draw either + if self.ppu_layout is not None: + needs_grid_decode = self.ppu_needs_grid_decode + else: + # None leaves it to the frame; only a proven True is kept + needs_grid_decode = True if self.ppu_needs_grid_decode else None + + # anchor_based None here means nothing conclusive; no box to draw return decode_ppu( outputs, self.width, @@ -961,8 +1248,11 @@ class DeepxDetector(DetectionApi): NMS_THRESHOLD, anchor_based=anchor_based, scale_count=self.ppu_scale_count or None, - needs_grid_decode=True if self.ppu_needs_grid_decode else None, + needs_grid_decode=needs_grid_decode, records=records, + # None leaves it to the frame; only a proven True is kept + centre_boxes=True if self.ppu_centre_boxes else None, + boxes=boxes, ) def decode_damoyolo(self, outputs: list[np.ndarray]) -> np.ndarray: diff --git a/frigate/test/test_deepx.py b/frigate/test/test_deepx.py index 875abe7e21..bfec5c117c 100644 --- a/frigate/test/test_deepx.py +++ b/frigate/test/test_deepx.py @@ -1,7 +1,10 @@ """Tests for the DEEPX detector.""" +import json import os +import struct import sys +import tempfile import unittest from unittest.mock import MagicMock, patch @@ -17,6 +20,7 @@ from frigate.detectors.plugins.deepx import ( PPU_RECORD_SIZE, DeepxDetector, DeepxDetectorConfig, + PpuLayout, YoloLayout, class_count, decode_damoyolo_raw, @@ -25,9 +29,11 @@ from frigate.detectors.plugins.deepx import ( decode_raw_nms_in_head, infer_yolo_layout, ppu_scale_count_lower_bound, + read_ppu_layout, resolve_device, rows_with_columns, validate_damoyolo_outputs, + validate_yolox_outputs, ) from frigate.util.runtime_deps import ArtifactKind @@ -56,6 +62,172 @@ def build_ppu_records(*records: np.ndarray) -> np.ndarray: return np.concatenate(records, axis=1) +# compile_config.ppu as DX-COM writes it for the two head kinds; the layer +# names inside are irrelevant to the reader, only the type is +ANCHOR_BASED_PPU = {"type": 0, "num_classes": 80, "activation": "Sigmoid"} +ANCHOR_FREE_PPU = {"type": 1, "num_classes": 80} + + +def write_dxnn(directory, ppu, layers, name="model.dxnn", table=True) -> str: + """A minimal .dxnn as DX-RT's parsers read it: the container header, a + compile_config carrying `ppu`, and either the PPU tensor table with one + entry per (layer, anchor) as a v8 file has, or with `table` False only + the rmap_info listing of the PPU output tensors, as a v7 file has. + `layers` is (grid_w, grid_h, entries) per scale, finest first; an + anchor-free scale has one entry, and grid_h 1 means a flattened + (1, cells, channels) tensor.""" + compile_config = json.dumps({"compile_version": "2.4.0", "ppu": ppu}).encode() + + if table: + part = bytearray(struct.pack(" 1: + name_ += f"_anchor_{anchor}" + shape = [1, grid_w, 127] if grid_h == 1 else [1, grid_h, grid_w, 128] + outputs.append({"name": name_, "shape": shape, "layout": "PPU_YOLO"}) + part = json.dumps({"inputs": [], "outputs": outputs}).encode() + + index = json.dumps( + { + "version": 8 if table else 7, + "signature": "DXNN", + "size": 8192, + "data": { + "compile_config": { + "type": "str", + "offset": 0, + "size": len(compile_config), + }, + "compiled_data": { + "M1A_4K": { + "npu_0": { + "rmap": {"type": "bytes", "offset": 0, "size": 0}, + "ppu" if table else "rmap_info": { + "type": "bytes" if table else "str", + "offset": len(compile_config), + "size": len(part), + }, + } + } + }, + }, + } + ).encode() + + path = os.path.join(directory, name) + with open(path, "wb") as model: + model.write(b"DXNN" + struct.pack("