diff --git a/docker/deepx/user_installation.sh b/docker/deepx/user_installation.sh old mode 100644 new mode 100755 diff --git a/docker/main/install_dxrt.sh b/docker/main/install_dxrt.sh old mode 100644 new mode 100755 index cd0aa04d0c..b9f534e91c --- a/docker/main/install_dxrt.sh +++ b/docker/main/install_dxrt.sh @@ -59,9 +59,15 @@ if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then exit 1 fi -apt-get -qq update -apt-get -qq install -y "${deb_file}" -rm -rf /var/lib/apt/lists/* +# Install with dpkg rather than apt. install_deps.sh finishes by purging gnupg +# and clearing /var/lib/apt/lists, so apt here would have to re-fetch about +# 10 MB of package indexes over the network before it could resolve anything, +# which is a build failure waiting to happen for no benefit. The package only +# depends on libc6, libgcc-s1, libncurses6, libstdc++6 and libtinfo6, all of +# which the deps stage already has, and its postinst does nothing but run +# ldconfig and print where the files went. dpkg fails loudly if a dependency +# ever goes missing. +dpkg -i "${deb_file}" # The package bundles a dx_engine wheel per cpython ABI tag. Ask dpkg what it # installed rather than hardcoding a path, so this keeps working if DEEPX moves diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run old mode 100644 new mode 100755 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish old mode 100644 new mode 100755 index fefd404cb2..8c35e27117 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish @@ -15,9 +15,19 @@ readonly service="dxrtd" echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" -# A clean exit means the service took itself down because no DEEPX hardware is -# present, which is the normal case on images without an NPU. Only a genuine -# failure should bring the container down. +# dxrtd takes itself down when the machine has no DEEPX NPU, which is the +# normal case: this service ships in the default image and most users do not +# have the hardware. That self-shutdown reaches this script as exit 256 / +# signal 15, indistinguishable from an operator stopping the container, so +# without the marker written by run every machine lacking a DEEPX NPU would +# have Frigate halted at startup. +if [[ -f /dev/shm/.dxrtd-no-hardware ]]; then + echo "[INFO] Service ${service} stopped because no DEEPX NPU is present; leaving the rest of Frigate running" + exit 0 +fi + +# Past this point the service really was running, so only a genuine failure +# should bring the container down. if [[ "${exit_code_service}" -eq 256 ]]; then if [[ "${exit_code_container}" -eq 0 ]]; then echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run old mode 100644 new mode 100755 index 27ff22217c..ac224a1c9e --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run @@ -14,9 +14,18 @@ exec 2>&1 # not have DEEPX hardware. if ! compgen -G "/dev/dxrt*" > /dev/null; then echo "[INFO] No DEEPX NPU device found, not starting dxrtd" + # Tell finish that this shutdown is deliberate. Taking the service down + # here arrives at finish as exit 256 / signal 15, which is exactly what an + # operator stopping the container looks like, so the marker is the only + # thing that tells the two apart. + touch /dev/shm/.dxrtd-no-hardware exec s6-svc -Od . fi +# Reaching here means there is hardware to serve, so clear any marker left by +# an earlier run before the device appeared. +rm -f /dev/shm/.dxrtd-no-hardware + # dxrtd is a system-wide singleton. If the host's own dxrt.service is still # running it will refuse to start, so surface that up front rather than # leaving the user to decode the daemon's own error. diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/dxrtd b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/dxrtd new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run index e3e86cb682..1cf7a1df31 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run @@ -4,7 +4,7 @@ set -o errexit -o nounset -o pipefail -dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync) +dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/dxrtd) mkdir -p "${dirs[@]}" diff --git a/docker/main/rootfs/labelmap/voc-20.txt b/docker/main/rootfs/labelmap/voc-20.txt new file mode 100644 index 0000000000..d04fc9e7ef --- /dev/null +++ b/docker/main/rootfs/labelmap/voc-20.txt @@ -0,0 +1,20 @@ +0 aeroplane +1 bicycle +2 bird +3 boat +4 bottle +5 bus +6 car +7 cat +8 chair +9 cow +10 diningtable +11 dog +12 horse +13 motorbike +14 person +15 pottedplant +16 sheep +17 sofa +18 train +19 tvmonitor diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 86da689ac5..ba264041f2 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -651,7 +651,9 @@ To configure a DEEPX detector, set the `type` attribute to `deepx` and follow th Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. -The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of YOLO variants alongside a JSON file describing how each was compiled. Pick a YOLO detection model, download the `.dxnn`, bind-mount it into the container, and point `model.path` at it. Note the variant you picked, as it determines the `model_format` value below. +The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of object detection models alongside a JSON file describing how each was compiled: YOLO variants (`model_type: yolo-generic`), SSD (`model_type: ssd`), and DAMO-YOLO (`model_type: damo-yolo`). Pick a model, download the `.dxnn`, bind-mount it into the container, and point `model.path` at it. Note which one you picked, since it determines the `model_type` and `model_format` values below. + +`model_type` is set on the detector rather than on the root `model` block. Frigate ignores a `model` block nested under a detector, and the root `model.model_type` defaults to `ssd`, so leaving it unset would silently decode a YOLO model with the SSD decoder and report no detections at all. The detector's own `model_type` defaults to `yolo-generic` and overrides the root value. Alternatively, compile your own model with the DX-COM compiler. @@ -659,47 +661,79 @@ Alternatively, compile your own model with the DX-COM compiler. #### Label maps -Models from the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. Frigate's default label map uses an extended 91-class scheme, and leaving it in place will cause detections to be reported as the wrong object type. +YOLO and DAMO-YOLO models from the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. SSD models are trained on Pascal VOC's 20-class label set instead, and need `labelmap_path` set to `/labelmap/voc-20.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type for any of these. -#### Model formats +#### YOLO model formats {#deepx-yolo-model-formats} -DX-COM preserves the detection head of the model it compiles, so the YOLO variant determines how the output has to be read. Set `model_format` to the variant your `.dxnn` file was compiled from: +For `model_type: yolo-generic`, DX-COM preserves the detection head of the model it compiles, so the head's tensor layout determines how the output has to be read. Set `model_format` to the layout your `.dxnn` file was compiled with: ```yaml detectors: deepx: type: deepx - model_format: yolov8 + model_format: anchor_free ``` -Supported values fall into three decode families: +Supported values are three decode families: -| `model_format` | Output layout | PPU support | -| ------------------------------------------ | ------------------------------------------------------------------------ | ----------- | -| `yolov5`, `yolov7` | Anchor-based, with a separate objectness score | Yes | -| `yolov8`, `yolov9`, `yolov11`, `yolov12` | Anchor-free, no objectness, boxes already in pixels | Yes | -| `yolov10`, `yolov26` | NMS runs in the detection head, output is final corner boxes | No | +| `model_format` | Output layout | PPU support | +| -------------- | -------------- | ----------- | +| `anchor` | Anchor-based, with a separate objectness score | Yes | +| `anchor_free` | Anchor-free, no objectness, boxes already in pixels | Yes | +| `nms_in_head` | NMS runs in the detection head, output is final corner boxes | No | -Leaving `model_format` unset falls back to inferring the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for YOLOv8 and newer. Setting it explicitly is recommended. +Leaving `model_format` at its `auto` default falls back to inferring the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for anchor-free heads. Setting it explicitly is recommended. Two mistakes this option prevents: - Decoding an anchor-free model with the anchor formula squares the box dimensions, producing boxes hundreds of thousands of pixels wide that clip to the entire frame. - Decoding an anchor-based model as anchor-free reads the objectness column as a class score, so confidences and class IDs are both wrong. -`yolov10` and `yolov26` run NMS inside the head, so `nms_threshold` does not apply to them. +`nms_in_head` models run NMS inside the head, so `nms_threshold` does not apply to them. -#### PPU models +#### SSD models -Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. - -Set `ppu: true` on the detector when using such a model: +For `model_type: ssd`, no `model_format` is needed. DX-COM keeps SSD's postprocessing inside the compiled model: the `.dxnn` runs the NPU head into a small CPU subgraph that concatenates the per-feature-map outputs, applies the softmax, and decodes the prior-box regression all the way to corner-form boxes. Only NMS is left for the host, so nothing about the backbone has to be named. This applies to all three ModelZoo backbones (MobileNetV1, MobileNetV2-Lite, VGG16) even though they emit different numbers of boxes (3000 and 8732). ```yaml detectors: deepx: type: deepx - model_format: yolov8 + model_type: ssd + +model: + width: 300 + height: 300 + labelmap_path: /labelmap/voc-20.txt +``` + +#### DAMO-YOLO models + +For `model_type: damo-yolo`, no `model_format` is needed — all four sizes (TinyNAS-L20T, S, M, L) share one detection head, differing only in backbone depth/width. Leave `model_format` at its `auto` default: + +```yaml +detectors: + deepx: + type: deepx + model_type: damo-yolo + +model: + width: 640 + height: 640 + labelmap_path: /labelmap/coco-80.txt +``` + +#### PPU models + +Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. PPU support is currently only available for `model_type: yolo-generic` — there is no confirmed PPU record layout for SSD or DAMO-YOLO yet, and setting `ppu: true` with either is rejected at startup. + +Set `ppu: true` on the detector when using a PPU-compiled yolo-generic model: + +```yaml +detectors: + deepx: + type: deepx + model_format: anchor_free ppu: true score_threshold: 0.25 nms_threshold: 0.45 @@ -707,24 +741,26 @@ detectors: `ppu` and `model_format` are independent: `ppu` says whether candidate selection already happened on the NPU, and `model_format` says how to read the boxes that come back. Both must match how the model was compiled. -`model_format` is required whenever `ppu: true` is set. A PPU record has the same fixed-width layout for every variant, so unlike the raw output path there is no shape to infer the layout from — leaving it unset is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch. +`model_format` is required whenever `ppu: true` is set. A PPU record has the same fixed-width layout for every yolo-generic variant, so unlike the raw output path there is no shape to infer the layout from — leaving it at `auto` is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch. -DX-COM cannot compile `yolov10` or `yolov26` models with PPU support, since their heads already produce final detections. Combining either with `ppu: true` is rejected at startup. +DX-COM cannot compile `nms_in_head` models with PPU support, since their heads already produce final detections. Combining the two is rejected at startup. #### Multiple NPUs -By default the detector binds to every NPU it can find. To pin a detector to specific devices, set `device_ids` to the indices you want it to use. This allows multiple detectors to be configured against separate modules: +`device_ids` defaults to `auto`, which binds the detector to every NPU it can find. To pin a detector to specific devices, set it to a comma-separated list of the indices you want it to use. This allows multiple detectors to be configured against separate modules: ```yaml detectors: deepx_0: type: deepx - device_ids: [0] + device_ids: "0" deepx_1: type: deepx - device_ids: [1] + device_ids: "1,2" ``` +A yaml list (`device_ids: [0, 1]`) is accepted as well, but the comma-separated string is what the config UI writes. + --- ## NVidia TensorRT Detector diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 2de31896e9..1180a76ecc 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -66,7 +66,7 @@ Frigate supports multiple different detectors that work on different types of ha - Runs best with tiny, small, or medium-size models - [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices. - - [Supports YOLO model architectures](../../configuration/object_detectors#deepx-npu) + - [Supports YOLO, SSD, and DAMO-YOLO model architectures](../../configuration/object_detectors#deepx-npu) - Runs best with tiny or small size models - Runs efficiently on low power hardware @@ -267,7 +267,7 @@ Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 mo DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com). -The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage. +The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO, SSD, and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and YOLO models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage. The DEEPX kernel driver has to be installed on the Docker host before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps. diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index 415b9389e2..29669dcc06 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -39,6 +39,7 @@ class InputDTypeEnum(str, Enum): class ModelTypeEnum(str, Enum): + damoyolo = "damo-yolo" dfine = "dfine" rfdetr = "rfdetr" ssd = "ssd" @@ -116,7 +117,7 @@ class ModelConfig(BaseModel): model_type: ModelTypeEnum = Field( default=ModelTypeEnum.ssd, title="Object Detection Model Type", - description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization.", + description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization.", ) _merged_labelmap: dict[int, str] | None = PrivateAttr() _colormap: dict[int, tuple[int, int, int]] = PrivateAttr() diff --git a/frigate/detectors/plugins/deepx.py b/frigate/detectors/plugins/deepx.py index c3d27c9a6b..f8286a7b02 100644 --- a/frigate/detectors/plugins/deepx.py +++ b/frigate/detectors/plugins/deepx.py @@ -8,7 +8,7 @@ from typing import Literal import cv2 import numpy as np -from pydantic import ConfigDict, Field, model_validator +from pydantic import ConfigDict, Field, field_validator, model_validator from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum @@ -18,29 +18,50 @@ logger = logging.getLogger(__name__) DETECTOR_KEY = "deepx" -# PPU-compiled models still use yolo-generic in the config; the `ppu` option is -# what selects the decode path, since ModelTypeEnum has no PPU-specific value. -supported_models = [ModelTypeEnum.yologeneric] - DXNN_DEVICES_ENV = "DXNN_DEVICES" +# Stands in for "let Frigate pick the devices". A plain empty string would be +# written back to the config as a deletion, so the no-op needs a real value. +DEVICE_IDS_AUTO = "auto" + + +class DeepxModelTypeEnum(str, Enum): + """The subset of Frigate's model types the DEEPX decoders can read. + + Frigate keeps `model_type` on the root `model` block and ignores a nested + one, so it is not selectable per detector in the config UI, and an + omitted value silently defaults to `ssd`. Repeating the supported values + here as a detector option makes the choice visible in the form; the + detector writes it back onto the merged model config at startup so the + rest of Frigate agrees with the decoder. + """ + + # PPU-compiled models still use yolo-generic; the `ppu` option is what + # selects the decode path, since there is no PPU-specific model type. + yologeneric = ModelTypeEnum.yologeneric.value + ssd = ModelTypeEnum.ssd.value + damoyolo = ModelTypeEnum.damoyolo.value + class ModelFormatEnum(str, Enum): - """The YOLO variant a .dxnn file was compiled from. + """How a .dxnn file's output is decoded. - DX-COM preserves the detection head of the source model, so the variant - determines how the output has to be read. Three families exist, and every - supported variant belongs to exactly one of them. + DX-COM preserves the detection head of the source model, so the exact + layout has to be told apart to read it. Values are grouped by which + `model_type` they apply to; a model_validator on DeepxDetectorConfig + enforces that a given value is only used with its matching type. """ - yolov5 = "yolov5" - yolov7 = "yolov7" - yolov8 = "yolov8" - yolov9 = "yolov9" - yolov10 = "yolov10" - yolov11 = "yolov11" - yolov12 = "yolov12" - yolov26 = "yolov26" + # "no explicit layout", and the default. model_type=yolo-generic infers + # the layout from the output shape; model_type=damo-yolo takes no layout + # at all. This is a real enum member rather than None so the config form + # always round-trips a value -- a null is written back as a deletion. + auto = "auto" + # model_type=yolo-generic: which detection-head tensor layout the .dxnn + # was compiled with. + anchor = "anchor" + anchor_free = "anchor_free" + nms_in_head = "nms_in_head" # Anchor-based heads. Raw output is (N, 5+C) with a separate objectness column, @@ -48,8 +69,7 @@ class ModelFormatEnum(str, Enum): # grid cell and anchor that produced them. ANCHOR_FORMATS = frozenset( { - ModelFormatEnum.yolov5, - ModelFormatEnum.yolov7, + ModelFormatEnum.anchor, } ) @@ -57,10 +77,7 @@ ANCHOR_FORMATS = frozenset( # column, and the box columns are already in input pixels. ANCHOR_FREE_FORMATS = frozenset( { - ModelFormatEnum.yolov8, - ModelFormatEnum.yolov9, - ModelFormatEnum.yolov11, - ModelFormatEnum.yolov12, + ModelFormatEnum.anchor_free, } ) @@ -68,12 +85,11 @@ ANCHOR_FREE_FORMATS = frozenset( # DX-COM cannot compile these with PPU support. NMS_IN_HEAD_FORMATS = frozenset( { - ModelFormatEnum.yolov10, - ModelFormatEnum.yolov26, + ModelFormatEnum.nms_in_head, } ) -# Anchor table for YOLOv5/YOLOv7 PPU models, keyed by stride. +# Anchor table for anchor-based PPU models, keyed by stride. PPU_ANCHORS = { 8: np.array([[10, 13], [16, 30], [33, 23]], dtype=np.float32), 16: np.array([[30, 61], [62, 45], [59, 119]], dtype=np.float32), @@ -89,6 +105,11 @@ PPU_SCORE_BYTES = (20, 24) PPU_LABEL_BYTES = (24, 28) +# DAMO-YOLO's ZeroHead is anchor-free at these three strides regardless of +# size (TinyNAS-L20T/S/M/L differ only in backbone depth/width). +DAMOYOLO_STRIDES = (8, 16, 32) + + def detect_device_count() -> int: """Count the NPU devices visible to this process. @@ -110,16 +131,32 @@ def detect_device_count() -> int: return len(devices) if devices else 1 -def resolve_devices(configured: list[int] | None) -> list[int]: +def parse_device_ids(value: str) -> list[int]: + """Parse a comma-separated list of NPU device indices. + + The `auto` sentinel and an empty value both mean "nothing was picked", + which leaves device selection to `resolve_devices`. + """ + if value.strip().lower() == DEVICE_IDS_AUTO: + return [] + + return [int(part) for part in value.split(",") if part.strip()] + + +def resolve_devices(configured: str) -> list[int]: """Resolve which NPU device indices this detector should bind to.""" if configured: - return configured + # already validated by DeepxDetectorConfig + devices = parse_device_ids(configured) + + if devices: + return devices env_value = os.environ.get(DXNN_DEVICES_ENV) if env_value: try: - devices = [int(d.strip()) for d in env_value.split(",") if d.strip()] + devices = parse_device_ids(env_value) except ValueError: logger.warning( "Ignoring unparsable %s=%s, detecting devices instead", @@ -162,6 +199,46 @@ def ppu_records(outputs: list[np.ndarray]) -> np.ndarray | None: return records +def _softmax(x: np.ndarray) -> np.ndarray: + """Numerically stable softmax over the last axis.""" + shifted = x - np.max(x, axis=-1, keepdims=True) + exp = np.exp(shifted) + return exp / np.sum(exp, axis=-1, keepdims=True) + + +def _split_box_and_class_outputs( + outputs: list[np.ndarray], +) -> tuple[np.ndarray, np.ndarray] | None: + """Split a 2-tensor raw output into (box tensor, class tensor) by shape. + + Used for model types where DX-COM's raw output tensor order isn't + confirmed. Whichever tensor's shape reads as box-like -- (..., 4), or + (..., 4, bins) for a per-side distribution -- is treated as boxes; the + other is class scores. Returns None if that can't be told apart, which + a class-score tensor with exactly 4 classes would also trigger. + """ + if len(outputs) != 2: + return None + + def is_box_shaped(t: np.ndarray) -> bool: + return (t.ndim == 3 and t.shape[-1] == 4) or (t.ndim == 4 and t.shape[-2] == 4) + + first, second = outputs[0], outputs[1] + first_is_box, second_is_box = is_box_shaped(first), is_box_shaped(second) + + if first_is_box and not second_is_box: + return first, second + if second_is_box and not first_is_box: + return second, first + + return None + + +def _shapes(outputs: list[np.ndarray]) -> list[tuple[int, ...]]: + """Output shapes, for diagnosing a decode path that can't read them.""" + return [np.shape(output) for output in outputs] + + def fill_detections( x_min: np.ndarray, y_min: np.ndarray, @@ -199,6 +276,18 @@ def run_nms( nms_threshold: float, ) -> np.ndarray: """Return the indices surviving NMS, in descending score order.""" + if logger.isEnabledFor(logging.DEBUG): + # Separates "the decode produced nothing" from "it produced boxes + # whose scores all fell under the threshold", which look identical + # from the empty detection array alone. + logger.debug( + "NMS input: %d boxes, best score %.4f, %d over threshold %.2f", + len(scores), + float(scores.max()) if len(scores) else 0.0, + int((scores >= score_threshold).sum()), + score_threshold, + ) + boxes_xywh = np.column_stack([x_min, y_min, box_w, box_h]) indices = cv2.dnn.NMSBoxes( boxes_xywh.tolist(), scores.tolist(), score_threshold, nms_threshold @@ -210,14 +299,14 @@ def run_nms( return np.array(indices).reshape(-1) -def decode_yolo_ppu_anchor( +def decode_ppu_anchor( outputs: list[np.ndarray], width: int, height: int, score_threshold: float, nms_threshold: float, ) -> np.ndarray: - """Decode PPU output from an anchor-based head (YOLOv5, YOLOv7). + """Decode PPU output from an anchor-based head. Models compiled with DEEPX's Post-Processing Unit perform candidate selection on the NPU and emit one fixed-width record per surviving @@ -254,8 +343,8 @@ def decode_yolo_ppu_anchor( anchor_w[mask] = anchors[anchor_idx[mask], 0] anchor_h[mask] = anchors[anchor_idx[mask], 1] - # YOLOv5-style anchor decode: the model emits sigmoid outputs that are - # rescaled against the grid cell and its anchor to recover pixel geometry. + # Anchor-based decode: the model emits sigmoid outputs that are rescaled + # against the grid cell and its anchor to recover pixel geometry. center_x = (boxes[:, 0] * 2.0 - 0.5 + grid_x) * stride center_y = (boxes[:, 1] * 2.0 - 0.5 + grid_y) * stride box_w = (boxes[:, 2] ** 2 * 4.0) * anchor_w @@ -279,14 +368,14 @@ def decode_yolo_ppu_anchor( ) -def decode_yolo_ppu_anchor_free( +def decode_ppu_anchor_free( outputs: list[np.ndarray], width: int, height: int, score_threshold: float, nms_threshold: float, ) -> np.ndarray: - """Decode PPU output from an anchor-free head (YOLOv8, v9, v11, v12). + """Decode PPU output from an anchor-free head. These heads have no anchors and no objectness, so the PPU writes the box directly as (center_x, center_y, width, height) in input pixels. The grid @@ -323,14 +412,14 @@ def decode_yolo_ppu_anchor_free( ) -def decode_yolo_raw_anchor( +def decode_raw_anchor( outputs: list[np.ndarray], width: int, height: int, score_threshold: float, nms_threshold: float, ) -> np.ndarray: - """Decode raw output from an anchor-based head (YOLOv5, YOLOv7). + """Decode raw output from an anchor-based head. The exported head emits (N, 5+C) rows of [center_x, center_y, width, height, objectness, class scores...] already in @@ -375,13 +464,13 @@ def decode_yolo_raw_anchor( ) -def decode_yolo_raw_nms_in_head( +def decode_raw_nms_in_head( outputs: list[np.ndarray], width: int, height: int, score_threshold: float, ) -> np.ndarray: - """Decode raw output from a head that already ran NMS (YOLOv10, YOLOv26). + """Decode raw output from a head that already ran NMS. These heads emit (N, 6) rows of [x_min, y_min, x_max, y_max, score, class] in input pixels, sorted by score. Running NMS again would be wasted work, @@ -415,34 +504,292 @@ def decode_yolo_raw_nms_in_head( ) +def decode_ssd_raw( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode SSD output: already-decoded boxes plus per-class probabilities. + + DX-COM keeps SSD's postprocessing inside the compiled model. The .dxnn + runs the NPU head into a small CPU subgraph that concatenates the + per-feature-map outputs, applies softmax to the confidences, and decodes + the prior-box regression (center variance 0.1, size variance 0.2, prior + table baked in as initializers) all the way to corner-form boxes. That + leaves only NMS for the host, which is what DEEPX's own ssdmv1, + ssdmv2lite and ssdvgg16 demos do -- their postprocessor has no prior + table, no variance constant, and no softmax. + + Expects two output tensors, order-independent: boxes shaped (1, N, 4) as + (x_min, y_min, x_max, y_max) normalized to [0, 1], and probabilities + shaped (1, N, num_classes + 1) with background at class 0. Background is + dropped and surviving labels are 0-indexed, to match a labelmap with no + background row. N is whatever the backbone produces (3000 for the + MobileNet variants, 8732 for VGG16) and never has to be known ahead of + time. + """ + split = _split_box_and_class_outputs(outputs) + + if split is None: + logger.debug( + "Could not identify SSD box/class outputs in %d tensor(s) with " + "shapes %s, skipping frame", + len(outputs), + _shapes(outputs), + ) + return np.zeros((20, 6), np.float32) + + boxes, confidences = split + + if ( + boxes.ndim != 3 + or confidences.ndim != 3 + or boxes.shape[0] == 0 + or boxes.shape[1] == 0 + ): + logger.debug( + "Unexpected SSD output rank: boxes %s, classes %s, skipping frame", + boxes.shape, + confidences.shape, + ) + return np.zeros((20, 6), np.float32) + + boxes = boxes[0] + confidences = confidences[0] + + if boxes.shape[0] != confidences.shape[0]: + logger.debug( + "SSD box count %d does not match class count %d, skipping frame", + boxes.shape[0], + confidences.shape[0], + ) + return np.zeros((20, 6), np.float32) + + # the model already softmaxed these; drop background (class 0) and + # relabel 0-indexed to match a labelmap with no background row + scores = confidences[:, 1:] + labels = np.argmax(scores, axis=1) + scores = scores[np.arange(len(labels)), labels] + + # normalized corner form -> input pixels, which is what NMS and + # fill_detections work in + x_min = boxes[:, 0] * width + y_min = boxes[:, 1] * height + x_max = boxes[:, 2] * width + y_max = boxes[:, 3] * height + + order = run_nms( + x_min, + y_min, + x_max - x_min, + y_max - y_min, + scores, + score_threshold, + nms_threshold, + ) + + return fill_detections( + x_min, y_min, x_max, y_max, scores, labels, width, height, order + ) + + +def _dfl_integral(distribution: np.ndarray) -> np.ndarray: + """Reduce a DFL distribution to an expected distance via weighted sum. + + Ported from tinyvision/DAMO-YOLO's Integral module (Apache-2.0): each of + the 4 box sides is predicted as a probability distribution over + `reg_max + 1` discrete distance bins rather than a single regressed + value; this recovers one scalar distance per side, in stride units. + """ + reg_max = distribution.shape[-1] - 1 + project = np.arange(reg_max + 1, dtype=np.float32) + return distribution @ project + + +def _damoyolo_center_priors( + width: int, height: int, strides: tuple[int, ...] +) -> np.ndarray: + """One (center_x, center_y, stride) row per grid cell across all scales, + in pixel space at the model's own input resolution. + + Ported from tinyvision/DAMO-YOLO's + ZeroHead.get_single_level_center_priors (Apache-2.0). + """ + rows = [] + + for stride in strides: + grid_h, grid_w = height // stride, width // stride + row_idx, col_idx = np.mgrid[0:grid_h, 0:grid_w] + centers_x = col_idx.ravel().astype(np.float32) * stride + centers_y = row_idx.ravel().astype(np.float32) * stride + strides_col = np.full(centers_x.shape, stride, dtype=np.float32) + rows.append(np.stack([centers_x, centers_y, strides_col], axis=1)) + + return np.concatenate(rows, axis=0) + + +def decode_damoyolo_raw( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, + strides: tuple[int, ...] = DAMOYOLO_STRIDES, +) -> np.ndarray: + """Decode DAMO-YOLO's raw output: sigmoid class scores + a DFL box head. + + Ported from tinyvision/DAMO-YOLO's ZeroHead (Apache-2.0). DAMO-YOLO is + anchor-free like YOLOv8 and newer, but unlike them its box regression is + a per-side Distribution Focal Loss histogram rather than a value already + in pixels, so it needs its own decode instead of reusing + decode_raw_anchor. Not yet verified against a real compiled model -- + see the two shape branches below for what DX-COM's export is assumed to + produce, and adjust if a real .dxnn's shapes don't match either one. + + Expects two output tensors, order-independent: class scores shaped + (1, N, num_classes), already sigmoid-activated, and a box output that is + either (1, N, 4, reg_max + 1) -- a raw DFL distribution per side, decoded + here via softmax + integral + stride-scaled distance-to-box -- or + already (1, N, 4), in case DX-COM's export folds that whole decode in + (the reference only ever produces a plain 4-wide box tensor as the + *result* of that decode, so this shape is treated as final + (x_min, y_min, x_max, y_max) pixel coordinates, not as a second + distance representation). + """ + split = _split_box_and_class_outputs(outputs) + + if split is None: + logger.debug( + "Could not identify DAMO-YOLO box/class outputs in %d tensor(s) with " + "shapes %s, skipping frame", + len(outputs), + _shapes(outputs), + ) + return np.zeros((20, 6), np.float32) + + box_output, cls_scores = split + + if cls_scores.ndim != 3 or cls_scores.shape[0] == 0 or cls_scores.shape[1] == 0: + return np.zeros((20, 6), np.float32) + + cls_scores = cls_scores[0] + num_priors = cls_scores.shape[0] + + if box_output.ndim == 4: + # raw per-side DFL distribution: softmax the bins, reduce to an + # expected distance (in stride units), then scale by each prior's + # own stride and apply it against that prior's center + distances = _dfl_integral(_softmax(box_output[0])) + + if distances.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO box count %d does not match class count %d, skipping frame", + distances.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + priors = _damoyolo_center_priors(width, height, strides) + + if priors.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO prior count %d does not match model output %d, " + "skipping frame", + priors.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + center_x, center_y, stride = priors[:, 0], priors[:, 1], priors[:, 2] + distances_px = distances * stride[:, None] + + x_min = center_x - distances_px[:, 0] + y_min = center_y - distances_px[:, 1] + x_max = center_x + distances_px[:, 2] + y_max = center_y + distances_px[:, 3] + elif box_output.ndim == 3 and box_output.shape[-1] == 4: + boxes = box_output[0] + + if boxes.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO box count %d does not match class count %d, skipping frame", + boxes.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + x_min, y_min, x_max, y_max = ( + boxes[:, 0], + boxes[:, 1], + boxes[:, 2], + boxes[:, 3], + ) + else: + logger.debug( + "Unexpected DAMO-YOLO box output shape %s, skipping frame", + box_output.shape, + ) + return np.zeros((20, 6), np.float32) + + labels = np.argmax(cls_scores, axis=1) + scores = cls_scores[np.arange(num_priors), labels] + + order = run_nms( + x_min, + y_min, + x_max - x_min, + y_max - y_min, + scores, + score_threshold, + nms_threshold, + ) + + return fill_detections( + x_min, y_min, x_max, y_max, scores, labels, width, height, order + ) + + class DeepxDetectorConfig(BaseDetectorConfig): """DEEPX NPU detector running .dxnn models via the DX-RT runtime.""" model_config = ConfigDict(title="DEEPX NPU") type: Literal[DETECTOR_KEY] - device_ids: list[int] | None = Field( - default=None, + device_ids: str = Field( + default=DEVICE_IDS_AUTO, title="DEEPX device indices", - description="NPU device indices this detector should bind to. Defaults " - "to auto-detection via the DXNN_DEVICES environment variable, then the " - "DX-RT device count, then a scan of /dev/dxrt*.", + description="Comma-separated NPU device indices this detector should " + 'bind to, e.g. "0,1". Leave as "auto" to detect them from the ' + "DXNN_DEVICES environment variable, then the DX-RT device count, then " + "a scan of /dev/dxrt*.", + ) + model_type: DeepxModelTypeEnum = Field( + default=DeepxModelTypeEnum.yologeneric, + title="Detection model architecture", + description="Which decoder reads the model's output. Overrides " + "model.model_type for this detector, whose default of ssd would " + "otherwise apply whenever the root model block leaves it unset.", ) ppu: bool = Field( default=False, title="PPU-compiled model", description="Set to true when the model was compiled with DEEPX's " "Post-Processing Unit support, which moves candidate selection onto " - "the NPU and changes the output format.", + "the NPU and changes the output format. Only supported for " + "model_type yolo-generic.", ) - model_format: ModelFormatEnum | None = Field( - default=None, - title="YOLO variant the model was compiled from", - description="Selects how the model's output is decoded. Leave unset to " - "infer the layout from the output shape, which cannot tell an " - "anchor-based head from an anchor-free one and is only reliable for " - "YOLOv8 and newer. Required when ppu is true, since a PPU record's " - "layout is identical for every variant and cannot be inferred.", + model_format: ModelFormatEnum = Field( + default=ModelFormatEnum.auto, + title="Detection head layout the model was compiled from", + description="Selects how the model's output is decoded. Only used " + 'for model_type yolo-generic: leave as "auto" to infer the layout ' + "from the output shape, which cannot tell an anchor-based head from " + "an anchor-free one and is only reliable for anchor-free heads. " + "Required when ppu is true, since a PPU record's layout is identical " + "for every yolo-generic variant and cannot be inferred. ssd and " + "damo-yolo each read one fixed layout and take no value here.", ) score_threshold: float = Field( default=0.25, @@ -453,29 +800,78 @@ class DeepxDetectorConfig(BaseDetectorConfig): title="NMS IoU threshold used when decoding detector output", ) + @field_validator("device_ids", mode="before") + @classmethod + def coerce_device_ids(cls, value): + """Accept the yaml list form as well as the comma-separated string + the config form writes, since a bare list is the natural thing to + hand-write. A blank entry folds back to the auto sentinel.""" + if isinstance(value, (list, tuple)): + value = ",".join(str(item) for item in value) + elif isinstance(value, int) and not isinstance(value, bool): + value = str(value) + + if value is None or (isinstance(value, str) and not value.strip()): + return DEVICE_IDS_AUTO + + return value + + @field_validator("device_ids") + @classmethod + def validate_device_ids(cls, value: str) -> str: + """Reject a device list that can't be parsed, so the error surfaces at + config load rather than when the detector process starts.""" + try: + parse_device_ids(value) + except ValueError: + raise ValueError( + f'device_ids "{value}" is not a comma-separated list of NPU ' + f'device indices, e.g. "0,1", or "{DEVICE_IDS_AUTO}".' + ) from None + + return value + + @field_validator("model_format", mode="before") + @classmethod + def coerce_model_format(cls, value): + """A blank or absent entry means "no explicit layout".""" + if value is None or (isinstance(value, str) and not value.strip()): + return ModelFormatEnum.auto + + return value + @model_validator(mode="after") def validate_ppu_is_supported(self): - """Reject a PPU/format pairing DX-COM cannot produce.""" - if self.ppu and self.model_format in NMS_IN_HEAD_FORMATS: + """Reject a PPU pairing DX-COM cannot produce, or that has no + confirmed PPU record layout yet.""" + if not self.ppu: + return self + + if self.model_format in NMS_IN_HEAD_FORMATS: raise ValueError( f"model_format '{self.model_format.value}' runs NMS in the " "detection head and cannot be compiled with PPU support by " "DX-COM. Set ppu to false." ) + if self.model_type is DeepxModelTypeEnum.damoyolo: + raise ValueError( + 'PPU is not yet supported for model_type "damo-yolo". Set ppu to false.' + ) + return self @model_validator(mode="after") def validate_ppu_requires_model_format(self): """Reject an unset model_format when ppu is enabled. - A PPU record is the same fixed-width layout for every variant, so - unlike the raw output path there is no shape to infer the layout - from. Leaving model_format unset here used to silently default to - the anchor-based decoder, which corrupts detections from anchor-free - PPU models (e.g. YOLOv8) instead of failing loudly. + A PPU record is the same fixed-width layout for every yolo-generic + variant, so unlike the raw output path there is no shape to infer + the layout from. Leaving model_format unset here used to silently + default to the anchor-based decoder, which corrupts detections from + anchor-free PPU models instead of failing loudly. """ - if self.ppu and self.model_format is None: + if self.ppu and self.model_format is ModelFormatEnum.auto: raise ValueError( "model_format must be set when ppu is true. The PPU output " "layout can't be inferred, and guessing wrong silently " @@ -484,6 +880,25 @@ class DeepxDetectorConfig(BaseDetectorConfig): return self + @model_validator(mode="after") + def validate_model_format_matches_model_type(self): + """Reject a model_format on a model_type that has no layout to pick. + + Only yolo-generic has variants the decoder can't tell apart. SSD and + DAMO-YOLO each read one fixed output layout, so any value other than + the sentinel is a config mistake rather than a choice.""" + if self.model_type is DeepxModelTypeEnum.yologeneric: + return self + + if self.model_format is not ModelFormatEnum.auto: + raise ValueError( + "model_format is not used for model_type " + f"{self.model_type.value} -- its output layout is fixed. " + 'Leave it as "auto".' + ) + + return self + class Deepx(DetectionApi): type_key = DETECTOR_KEY @@ -500,11 +915,13 @@ class Deepx(DetectionApi): super().__init__(config) - if config.model.model_type not in supported_models: - raise ValueError( - f'Model type "{config.model.model_type}" is not supported by the ' - "deepx detector" - ) + # Frigate ignores a nested `model` block, so model.model_type carries + # the root value -- `ssd` whenever it was left unset. The detector's + # own model_type is the one the user actually picked, so it wins, and + # is written back for everything downstream that reads the model + # config (the events table, /api/config). + self.model_type = ModelTypeEnum(config.model_type.value) + config.model.model_type = self.model_type model_path = config.model.path @@ -528,20 +945,45 @@ class Deepx(DetectionApi): self.score_threshold = config.score_threshold self.nms_threshold = config.nms_threshold + self.logged_layout = False + logger.info( - "DEEPX decoding %s output as %s", + "DEEPX decoding %s output for model_type %s as %s", "PPU" if self.ppu else "raw", - self.model_format.value if self.model_format else "auto-detected", + self.model_type.value, + "auto-detected" + if self.model_format is ModelFormatEnum.auto + else self.model_format.value, ) def decode(self, outputs: list[np.ndarray]) -> np.ndarray: - """Decode raw or PPU output according to the configured YOLO variant.""" + """Decode raw or PPU output according to the configured model type + and detection head layout.""" + if self.model_type == ModelTypeEnum.ssd: + return decode_ssd_raw( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + if self.model_type == ModelTypeEnum.damoyolo: + return decode_damoyolo_raw( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + # model_type is yolo-generic from here on if self.ppu: # Config validation guarantees model_format is set to an anchor # or anchor-free format whenever ppu is true, since PPU records # give no other way to tell the two apart. if self.model_format in ANCHOR_FREE_FORMATS: - return decode_yolo_ppu_anchor_free( + return decode_ppu_anchor_free( outputs, self.width, self.height, @@ -549,7 +991,7 @@ class Deepx(DetectionApi): self.nms_threshold, ) - return decode_yolo_ppu_anchor( + return decode_ppu_anchor( outputs, self.width, self.height, @@ -558,7 +1000,7 @@ class Deepx(DetectionApi): ) if self.model_format in NMS_IN_HEAD_FORMATS: - return decode_yolo_raw_nms_in_head( + return decode_raw_nms_in_head( outputs, self.width, self.height, self.score_threshold ) @@ -566,7 +1008,7 @@ class Deepx(DetectionApi): # multi-part output is a set of raw feature maps, which the shared # helper already decodes with the same anchor formula. if self.model_format in ANCHOR_FORMATS and len(outputs) == 1: - return decode_yolo_raw_anchor( + return decode_raw_anchor( outputs, self.width, self.height, @@ -583,4 +1025,41 @@ class Deepx(DetectionApi): if not isinstance(outputs, list): outputs = [outputs] - return self.decode(outputs) + detections = self.decode(outputs) + + if not self.logged_layout: + self.logged_layout = True + self.log_layout(tensor_input, outputs) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "DEEPX decoded %d detections, best score %.4f", + int((detections[:, 1] > 0).sum()), + float(detections[:, 1].max()), + ) + + return detections + + def log_layout(self, tensor_input, outputs: list[np.ndarray]) -> None: + """Log the model's real input and output layout, once per start. + + Every decode path is a guess about the layout DX-COM compiled the + detection head into, and a wrong guess returns an empty detection + array rather than failing. These shapes and value ranges are the + first thing needed to tell those apart, so they go out at info + level: waiting to reproduce them with debug logging enabled costs a + restart, and this is one line per tensor per detector start. + """ + tensors = [("input", tensor_input)] + tensors += [(f"output[{i}]", out) for i, out in enumerate(outputs)] + + for name, tensor in tensors: + array = np.asarray(tensor) + logger.info( + "DEEPX %s: shape=%s dtype=%s min=%s max=%s", + name, + array.shape, + array.dtype, + array.min() if array.size else "n/a", + array.max() if array.size else "n/a", + ) diff --git a/frigate/test/test_deepx.py b/frigate/test/test_deepx.py index 5902ef805e..d2ecb55f6b 100644 --- a/frigate/test/test_deepx.py +++ b/frigate/test/test_deepx.py @@ -5,18 +5,25 @@ import unittest import numpy as np from pydantic import ValidationError +from frigate.detectors.detector_config import ModelTypeEnum from frigate.detectors.plugins.deepx import ( ANCHOR_FORMATS, ANCHOR_FREE_FORMATS, + DEVICE_IDS_AUTO, NMS_IN_HEAD_FORMATS, PPU_ANCHORS, PPU_RECORD_SIZE, DeepxDetectorConfig, + DeepxModelTypeEnum, ModelFormatEnum, - decode_yolo_ppu_anchor, - decode_yolo_ppu_anchor_free, - decode_yolo_raw_anchor, - decode_yolo_raw_nms_in_head, + decode_damoyolo_raw, + decode_ppu_anchor, + decode_ppu_anchor_free, + decode_raw_anchor, + decode_raw_nms_in_head, + decode_ssd_raw, + parse_device_ids, + resolve_devices, ) @@ -53,7 +60,7 @@ class TestDeepxPpuDecode(unittest.TestCase): ).reshape(1, 1, PPU_RECORD_SIZE) ] - detections = decode_yolo_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) anchor_w, anchor_h = PPU_ANCHORS[8][0] # center = (0.5 * 2 - 0.5 + 10) * 8, size = (0.5 ** 2 * 4) * anchor @@ -85,7 +92,7 @@ class TestDeepxPpuDecode(unittest.TestCase): label=0, ).reshape(1, 1, PPU_RECORD_SIZE) ] - return decode_yolo_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + return decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) small = decode(0) large = decode(2) @@ -108,7 +115,7 @@ class TestDeepxPpuDecode(unittest.TestCase): ).reshape(1, 1, PPU_RECORD_SIZE) ] - detections = decode_yolo_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) self.assertTrue(np.all(detections == 0)) @@ -128,29 +135,31 @@ class TestDeepxPpuDecode(unittest.TestCase): ] ) - detections = decode_yolo_ppu_anchor( + detections = decode_ppu_anchor( [records.reshape(1, 40, PPU_RECORD_SIZE)], 640, 640, 0.25, 0.45 ) self.assertEqual(detections.shape, (20, 6)) def test_returns_empty_detections_for_no_output(self): - self.assertTrue(np.all(decode_yolo_ppu_anchor([], 640, 640, 0.25, 0.45) == 0)) + self.assertTrue(np.all(decode_ppu_anchor([], 640, 640, 0.25, 0.45) == 0)) def test_returns_empty_detections_for_an_unexpected_record_width(self): # a non-PPU model would produce a different record width outputs = [np.zeros((1, 4, 16), dtype=np.uint8)] - self.assertTrue( - np.all(decode_yolo_ppu_anchor(outputs, 640, 640, 0.25, 0.45) == 0) - ) + self.assertTrue(np.all(decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) == 0)) class TestDeepxModelFormats(unittest.TestCase): - def test_every_format_belongs_to_exactly_one_family(self): + def test_every_yolo_format_belongs_to_exactly_one_family(self): families = [ANCHOR_FORMATS, ANCHOR_FREE_FORMATS, NMS_IN_HEAD_FORMATS] + yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS for fmt in ModelFormatEnum: + if fmt not in yolo_formats: + continue + matches = [family for family in families if fmt in family] self.assertEqual( len(matches), 1, f"{fmt.value} must be in exactly one family" @@ -161,6 +170,24 @@ class TestDeepxModelFormats(unittest.TestCase): self.assertFalse(ANCHOR_FORMATS & NMS_IN_HEAD_FORMATS) self.assertFalse(ANCHOR_FREE_FORMATS & NMS_IN_HEAD_FORMATS) + def test_every_format_belongs_to_a_yolo_family(self): + # apart from the `auto` sentinel, every ModelFormatEnum member is a + # yolo-generic head shape -- nothing should be unclassified + yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS + + for fmt in ModelFormatEnum: + if fmt is ModelFormatEnum.auto: + continue + + self.assertIn(fmt, yolo_formats, f"{fmt.value} is not classified") + + def test_auto_is_in_no_family(self): + """`auto` means "no explicit layout", so every decode branch keyed off + a family must fall through to shape-based inference.""" + self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FORMATS) + self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FREE_FORMATS) + self.assertNotIn(ModelFormatEnum.auto, NMS_IN_HEAD_FORMATS) + class TestDeepxDetectorConfig(unittest.TestCase): def test_ppu_without_model_format_is_rejected(self): @@ -172,17 +199,118 @@ class TestDeepxDetectorConfig(unittest.TestCase): def test_ppu_with_anchor_free_model_format_is_accepted(self): config = DeepxDetectorConfig( - type="deepx", ppu=True, model_format=ModelFormatEnum.yolov8 + type="deepx", ppu=True, model_format=ModelFormatEnum.anchor_free ) - self.assertEqual(config.model_format, ModelFormatEnum.yolov8) + self.assertEqual(config.model_format, ModelFormatEnum.anchor_free) def test_raw_without_model_format_is_still_accepted(self): # the raw path falls back to shape-based inference, so this remains # optional when ppu is not set config = DeepxDetectorConfig(type="deepx") - self.assertIsNone(config.model_format) + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_an_omitted_model_format_folds_to_auto(self): + """The config form round-trips a value for every field, so a blank or + null entry has to land on the sentinel rather than stay None.""" + for value in (None, "", " "): + with self.subTest(value=value): + config = DeepxDetectorConfig(type="deepx", model_format=value) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_ppu_with_damoyolo_model_type_is_rejected(self): + """No confirmed PPU record layout exists for DAMO-YOLO yet.""" + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + ppu=True, + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.damoyolo, + ) + + def test_ssd_model_type_takes_no_model_format(self): + """SSD's layout is fixed: DX-COM keeps the softmax and prior decode + inside the .dxnn, so no backbone has to be named.""" + config = DeepxDetectorConfig(type="deepx", model_type=DeepxModelTypeEnum.ssd) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_ssd_model_type_rejects_a_model_format(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.ssd, + ) + + def test_damoyolo_model_type_rejects_a_model_format(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.damoyolo, + ) + + def test_damoyolo_model_type_without_model_format_is_accepted(self): + config = DeepxDetectorConfig( + type="deepx", + model_type=DeepxModelTypeEnum.damoyolo, + ) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + +class TestDeepxDeviceIds(unittest.TestCase): + def test_a_comma_separated_string_is_kept(self): + config = DeepxDetectorConfig(type="deepx", device_ids="0,1") + + self.assertEqual(config.device_ids, "0,1") + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_a_yaml_list_is_accepted(self): + """A bare list is the natural thing to hand-write, so it is folded + into the same comma-separated form the config UI writes.""" + config = DeepxDetectorConfig(type="deepx", device_ids=[0, 1]) + + self.assertEqual(config.device_ids, "0,1") + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_a_single_index_is_accepted(self): + config = DeepxDetectorConfig(type="deepx", device_ids=1) + + self.assertEqual(config.device_ids, "1") + self.assertEqual(resolve_devices(config.device_ids), [1]) + + def test_surrounding_whitespace_is_tolerated(self): + config = DeepxDetectorConfig(type="deepx", device_ids=" 0 , 1 ") + + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_an_unparsable_value_is_rejected_at_config_load(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig(type="deepx", device_ids="gpu0") + + def test_unset_leaves_device_selection_to_auto_detection(self): + config = DeepxDetectorConfig(type="deepx") + + self.assertEqual(config.device_ids, DEVICE_IDS_AUTO) + self.assertEqual(parse_device_ids(config.device_ids), []) + + def test_a_blank_entry_folds_to_auto(self): + """The config form round-trips a value for every field, so a blank or + null entry has to land on the sentinel rather than stay None.""" + for value in (None, "", " ", []): + with self.subTest(value=value): + config = DeepxDetectorConfig(type="deepx", device_ids=value) + + self.assertEqual(config.device_ids, DEVICE_IDS_AUTO) + + def test_auto_is_case_insensitive(self): + config = DeepxDetectorConfig(type="deepx", device_ids="AUTO") + + self.assertEqual(parse_device_ids(config.device_ids), []) class TestDeepxPpuAnchorFreeDecode(unittest.TestCase): @@ -201,7 +329,7 @@ class TestDeepxPpuAnchorFreeDecode(unittest.TestCase): ) outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)] - detections = decode_yolo_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) + detections = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) # x: 320 +/- 32 -> 288..352, y: 160 +/- 16 -> 144..176 self.assertAlmostEqual(detections[0][2], 144 / 640, places=5) @@ -222,8 +350,8 @@ class TestDeepxPpuAnchorFreeDecode(unittest.TestCase): ) outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)] - anchor_free = decode_yolo_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) - anchor = decode_yolo_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + anchor_free = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) + anchor = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) # the anchor decode squares the width, saturating the clip to the frame self.assertEqual(anchor[0][5], 1.0) @@ -247,7 +375,7 @@ class TestDeepxRawAnchorDecode(unittest.TestCase): [(320.0, 320.0, 40.0, 80.0, 0.8, 3, 0.5)] ) - detections = decode_yolo_raw_anchor(outputs, 640, 640, 0.25, 0.45) + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) self.assertEqual(detections[0][0], 3) self.assertAlmostEqual(detections[0][1], 0.4, places=5) @@ -259,7 +387,7 @@ class TestDeepxRawAnchorDecode(unittest.TestCase): [(320.0, 320.0, 40.0, 80.0, 0.4, 3, 0.5)] ) - detections = decode_yolo_raw_anchor(outputs, 640, 640, 0.25, 0.45) + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) self.assertTrue(np.all(detections == 0)) @@ -268,7 +396,7 @@ class TestDeepxRawAnchorDecode(unittest.TestCase): [(320.0, 160.0, 64.0, 32.0, 1.0, 0, 1.0)] ) - detections = decode_yolo_raw_anchor(outputs, 640, 640, 0.25, 0.45) + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) self.assertAlmostEqual(detections[0][2], 144 / 640, places=5) self.assertAlmostEqual(detections[0][3], 288 / 640, places=5) @@ -289,7 +417,7 @@ class TestDeepxRawNmsInHeadDecode(unittest.TestCase): dtype=np.float32, ) - detections = decode_yolo_raw_nms_in_head([out], 640, 640, 0.25) + detections = decode_raw_nms_in_head([out], 640, 640, 0.25) self.assertEqual(detections[0][0], 2) self.assertAlmostEqual(detections[0][1], 0.9, places=5) @@ -308,7 +436,7 @@ class TestDeepxRawNmsInHeadDecode(unittest.TestCase): dtype=np.float32, ) - detections = decode_yolo_raw_nms_in_head([out], 640, 640, 0.25) + detections = decode_raw_nms_in_head([out], 640, 640, 0.25) self.assertAlmostEqual(detections[0][1], 0.9, places=5) self.assertTrue(np.all(detections[1] == 0)) @@ -316,8 +444,235 @@ class TestDeepxRawNmsInHeadDecode(unittest.TestCase): def test_returns_empty_detections_for_an_empty_output(self): out = np.zeros((1, 0, 6), dtype=np.float32) - self.assertTrue(np.all(decode_yolo_raw_nms_in_head([out], 640, 640, 0.25) == 0)) + self.assertTrue(np.all(decode_raw_nms_in_head([out], 640, 640, 0.25) == 0)) + + +class TestDeepxSsdDecode(unittest.TestCase): + """The .dxnn emits final boxes and probabilities, so these feed the + decoder what a real compiled SSD produces: corner-form boxes normalized + to [0, 1] and post-softmax confidences with background at column 0.""" + + def build_ssd_output(self, box_index, box, class_id, prob, num_boxes=3000): + num_classes = 21 # 20 VOC classes + background + + boxes = np.zeros((1, num_boxes, 4), dtype=np.float32) + boxes[0, box_index] = box + + # every other box is background, as the model would report it + scores = np.zeros((1, num_boxes, num_classes), dtype=np.float32) + scores[0, :, 0] = 1.0 + scores[0, box_index, 0] = 1.0 - prob + scores[0, box_index, class_id] = prob + + return [boxes, scores] + + def test_decodes_an_already_decoded_box(self): + outputs = self.build_ssd_output( + box_index=7, box=(0.1, 0.2, 0.5, 0.8), class_id=5, prob=0.9 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + # boxes pass through untouched apart from the corner reorder, and + # the background class is dropped so labels shift down by one + self.assertEqual(detections[0][0], 4) + self.assertAlmostEqual(float(detections[0][1]), 0.9, places=5) + self.assertAlmostEqual(float(detections[0][2]), 0.2, places=5) # y_min + self.assertAlmostEqual(float(detections[0][3]), 0.1, places=5) # x_min + self.assertAlmostEqual(float(detections[0][4]), 0.8, places=5) # y_max + self.assertAlmostEqual(float(detections[0][5]), 0.5, places=5) # x_max + + def test_confidences_are_not_softmaxed_again(self): + """A second softmax over 21 already-normalized probabilities pulls + every score toward 1/21, which is what silently dropped every real + detection under the default threshold.""" + outputs = self.build_ssd_output( + box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=15, prob=0.6 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertAlmostEqual(float(detections[0][1]), 0.6, places=5) + + def test_any_box_count_is_accepted(self): + """3000 for the MobileNet backbones, 8732 for VGG16 -- the decoder + reads whatever the model emits rather than a per-backbone table.""" + for num_boxes in (3000, 8732): + with self.subTest(num_boxes=num_boxes): + outputs = self.build_ssd_output( + box_index=num_boxes - 1, + box=(0.0, 0.0, 1.0, 1.0), + class_id=1, + prob=0.8, + num_boxes=num_boxes, + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertEqual(detections[0][0], 0) + self.assertAlmostEqual(float(detections[0][1]), 0.8, places=5) + + def test_output_order_does_not_matter(self): + boxes, scores = self.build_ssd_output( + box_index=1, box=(0.2, 0.2, 0.6, 0.6), class_id=3, prob=0.7 + ) + + forward = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45) + reversed_ = decode_ssd_raw([scores, boxes], 300, 300, 0.25, 0.45) + + np.testing.assert_array_equal(forward, reversed_) + + def test_low_confidence_detection_is_dropped(self): + outputs = self.build_ssd_output( + box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=3, prob=0.05 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.5, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_outputs_cannot_be_told_apart(self): + # two tensors that both look class-shaped (last dim != 4) + outputs = [ + np.zeros((1, 10, 21), dtype=np.float32), + np.zeros((1, 10, 21), dtype=np.float32), + ] + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_box_and_class_counts_disagree(self): + boxes = np.zeros((1, 10, 4), dtype=np.float32) + scores = np.zeros((1, 12, 21), dtype=np.float32) + + detections = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + +class TestDeepxDamoyoloDecode(unittest.TestCase): + def total_priors(self, width, height, strides): + return sum((height // s) * (width // s) for s in strides) + + def test_decodes_a_detection_from_already_decoded_boxes(self): + """box_output shaped (1, N, 4): DX-COM's export is assumed to have + already applied the full distance-to-box decode, so these are final + (x_min, y_min, x_max, y_max) pixel coordinates, used as-is.""" + width = height = 640 + num_priors = 100 + prior_index = 5 + + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, prior_index, 7] = 0.9 + + box_output = np.zeros((1, num_priors, 4), dtype=np.float32) + box_output[0, prior_index] = [100.0, 120.0, 140.0, 160.0] + + detections = decode_damoyolo_raw( + [cls_scores, box_output], width, height, 0.25, 0.45 + ) + + self.assertEqual(detections[0][0], 7) + self.assertAlmostEqual(detections[0][1], 0.9, places=5) + self.assertAlmostEqual(detections[0][2], 120 / height, places=5) + self.assertAlmostEqual(detections[0][3], 100 / width, places=5) + self.assertAlmostEqual(detections[0][4], 160 / height, places=5) + self.assertAlmostEqual(detections[0][5], 140 / width, places=5) + + def test_decodes_a_detection_from_a_dfl_distribution(self): + """box_output shaped (1, N, 4, bins): a raw per-side DFL histogram, + needing softmax + integral + stride-scaled distance-to-box here.""" + width = height = 640 + strides = (8, 16, 32) + grid_w = width // 8 + prior_row, prior_col = 10, 10 + prior_index = prior_row * grid_w + prior_col + num_priors = self.total_priors(width, height, strides) + + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, prior_index, 9] = 0.8 + + # reg_max=1 (2 bins): put ~all softmax mass on bin index 1, so the + # integral (weighted sum against [0, 1]) comes out to ~1.0 stride + # unit on every side + box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32) + box_output[0, prior_index, :, 0] = -30.0 + box_output[0, prior_index, :, 1] = 30.0 + + detections = decode_damoyolo_raw( + [cls_scores, box_output], width, height, 0.25, 0.45, strides + ) + + # stride-8 grid cell (10, 10) centers at pixel (80, 80); ~1 + # stride-unit distance on every side -> an 8px box around it + self.assertEqual(detections[0][0], 9) + self.assertAlmostEqual(detections[0][1], 0.8, places=5) + self.assertAlmostEqual(detections[0][2], 72 / height, places=3) + self.assertAlmostEqual(detections[0][3], 72 / width, places=3) + self.assertAlmostEqual(detections[0][4], 88 / height, places=3) + self.assertAlmostEqual(detections[0][5], 88 / width, places=3) + + def test_returns_empty_detections_below_score_threshold(self): + num_priors = 10 + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.1 + + box_output = np.zeros((1, num_priors, 4), dtype=np.float32) + box_output[0, 0] = [10.0, 10.0, 20.0, 20.0] + + detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.5, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_dfl_prior_count_does_not_match_the_grid( + self, + ): + # a DFL-shaped box output with far fewer priors than the (8, 16, 32) + # strides actually produce for a 640x640 input + num_priors = 5 + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.9 + box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32) + + detections = decode_damoyolo_raw( + [cls_scores, box_output], 640, 640, 0.25, 0.45, (8, 16, 32) + ) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_box_and_class_counts_disagree(self): + cls_scores = np.zeros((1, 10, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.9 + box_output = np.zeros((1, 7, 4), dtype=np.float32) + + detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_for_wrong_output_count(self): + outputs = [np.zeros((1, 10, 80), dtype=np.float32)] + + detections = decode_damoyolo_raw(outputs, 640, 640, 0.25, 0.45, (8, 16, 32)) + + self.assertTrue(np.all(detections == 0)) if __name__ == "__main__": unittest.main() + + +class TestDeepxModelType(unittest.TestCase): + def test_model_type_defaults_to_yolo_generic(self): + """Frigate's own model_type default is ssd, which silently picked the + SSD decoder for anyone who left it unset.""" + config = DeepxDetectorConfig(type="deepx") + + self.assertIs(config.model_type, DeepxModelTypeEnum.yologeneric) + + def test_model_type_values_match_frigate_model_types(self): + """The detector writes its choice back onto model.model_type, so every + value has to round-trip through ModelTypeEnum.""" + for value in DeepxModelTypeEnum: + with self.subTest(value=value): + self.assertEqual(ModelTypeEnum(value.value).value, value.value)