diff --git a/CODEOWNERS b/CODEOWNERS index c37041c2c3..e38d8e3ea7 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,3 +5,4 @@ /docker/rockchip/ @MarcA711 /docker/rocm/ @harakas /docker/hailo8l/ @spanner3003 +/docker/deepx/ @sixfab diff --git a/docker/deepx/user_installation.sh b/docker/deepx/user_installation.sh new file mode 100755 index 0000000000..12c9508b9c --- /dev/null +++ b/docker/deepx/user_installation.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +# Installs the DEEPX NPU kernel driver on the Docker host. +# +# The driver has to be built and loaded on the host rather than in the Frigate +# container: containers share the host kernel, so a kernel module cannot be +# loaded from inside one. Once loaded it exposes the NPU as /dev/dxrt* device +# nodes, which are passed through to the container (see the DEEPX section of +# the installation docs). +# +# A DEEPX install has three separately versioned pieces that have to agree: +# +# DX-RT runtime v3.4.2 the Frigate image, docker/main/install_dxrt.sh +# Kernel driver v2.6.0 this file (provides dxrt_driver and dx_dma) +# NPU firmware v2.7.4 the module itself, flashed from the host +# +# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the +# versions pinned here sit above that floor. A mismatch is the most common +# cause of inference requests that are accepted but never complete, and it +# fails silently rather than at startup. This script only installs the driver: +# reading or flashing the firmware version requires the DX-RT runtime on the +# host, which conflicts with the copy Frigate runs in its container, so that is +# a separate manual step covered in the installation docs. +# +# DEEPX NPU support in Frigate is developed and maintained by Sixfab +# (https://sixfab.com). + +set -euo pipefail + +driver_version="v2.6.0" +# Commit the tag above resolves to, pinned so a later retag or repoint of +# v2.6.0 upstream can't silently swap in different source between review and +# build, before it's compiled and installed as root. DEEPX doesn't sign tags +# or publish checksums for this repo, so an immutable commit is the strongest +# verification available. Update both together when bumping driver_version: +# git ls-remote https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git "refs/tags/" +driver_commit="7074748e7104f470b02f517583abba652b3f05fa" +firmware_version="v2.7.4" + +sudo apt-get update +sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils + +if ! lspci -d 1ff4: | grep -q .; then + echo "No DEEPX device found on the PCIe bus (lspci -d 1ff4:)." + echo "Check that the module is seated correctly before continuing." + exit 1 +fi + +# Fetch the pinned commit directly (GitHub allows fetching a public repo by +# SHA, not just by branch/tag) rather than cloning "${driver_version}", so +# driver_commit above is what actually determines the source, not a ref that +# could move after this script was reviewed. +mkdir dx_rt_npu_linux_driver +cd dx_rt_npu_linux_driver +git init -q +git remote add origin https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git +git fetch --depth 1 origin "${driver_commit}" +git checkout -q FETCH_HEAD + +fetched_commit=$(git rev-parse HEAD) +if [[ "${fetched_commit}" != "${driver_commit}" ]]; then + echo "Fetched commit ${fetched_commit} does not match pinned driver_commit ${driver_commit}." + echo "Refusing to build unverified driver source." + exit 1 +fi + +cd modules + +# --reload unloads any running modules and loads the newly built ones, so the +# NPU is usable without a reboot. +sudo ./build.sh -c install --reload + +sudo depmod -A + +# dx_dma is the PCIe transport, dxrt_driver is the NPU driver on top of it. +for module in dx_dma dxrt_driver; do + if ! sudo modprobe "${module}"; then + echo "Unable to load the ${module} kernel module, common reasons are:" + echo "- Secure Boot is enabled and is rejecting the unsigned module." + echo "- The running kernel does not match the installed linux-headers." + exit 1 + fi +done + +if ! compgen -G "/dev/dxrt*" > /dev/null; then + echo "Modules loaded but no /dev/dxrt* device node appeared." + echo "Run ./sanity_check.sh from the driver repo to diagnose." + exit 1 +fi + +# dxrtd runs inside the Frigate container and may only run once per system, so +# a host copy left over from a runtime install has to be disabled. +if systemctl list-unit-files dxrt.service &> /dev/null; then + echo "Disabling the host dxrt.service so the container can run dxrtd." + sudo systemctl disable --now dxrt.service +fi + +echo "DEEPX driver installation complete." +echo "Driver version: $(modinfo -F version dxrt_driver)" +echo "Device node(s): $(echo /dev/dxrt*)" +echo +echo "This driver expects NPU firmware ${firmware_version}. The firmware version" +echo "can only be read with the DX-RT runtime installed on the host, so check it" +echo "before starting Frigate, then remove the host runtime: Frigate runs its own" +echo "copy of dxrtd and only one may run per system." diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index bf17146e30..41bc6b3e3a 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -285,6 +285,10 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/axcl" RUN --mount=type=bind,source=docker/main/install_memryx.sh,target=/deps/install_memryx.sh \ bash -c "bash /deps/install_memryx.sh" +# Install DEEPX DX-RT runtime (dxrtd daemon + dx_engine python bindings) +RUN --mount=type=bind,source=docker/main/install_dxrt.sh,target=/deps/install_dxrt.sh \ + /deps/install_dxrt.sh + COPY --from=deps-rootfs / / RUN ldconfig diff --git a/docker/main/install_dxrt.sh b/docker/main/install_dxrt.sh new file mode 100755 index 0000000000..b9f534e91c --- /dev/null +++ b/docker/main/install_dxrt.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -euxo pipefail + +# DEEPX DX-RT runtime (dxrtd daemon + the dx_engine python bindings) used by +# the `deepx` detector plugin. +# +# A DEEPX install has three separately versioned pieces that have to agree: +# +# DX-RT runtime v3.4.2 this file, installed into the Frigate image +# Kernel driver v2.6.0 the host, docker/deepx/user_installation.sh +# NPU firmware v2.7.4 the module itself, flashed from the host +# +# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the +# versions above sit just over that floor. The runtime is pinned rather than +# resolved to "latest" so image builds stay reproducible, and so there is a +# known version for the host side to match. A mismatch usually presents as +# inference requests that are accepted but never complete, not as a startup +# error, so bump all three together and keep the table in +# docs/docs/frigate/installation.md in sync. +dxrt_version="v3.4.2" + +# SHA-256 of the .deb pinned above, one per architecture Frigate builds for. +# DEEPX doesn't publish checksums or sign this release, so these are computed +# once from a vetted download and hardcoded here as the actual integrity +# check: a download that doesn't match one of these hashes is rejected before +# it reaches apt-get, regardless of whether the mismatch came from the v3.4.2 +# tag being retargeted or the file being replaced upstream. Recompute both +# (`sha256sum libdxrt-bin_*.deb`) when bumping dxrt_version. +declare -A dxrt_sha256=( + [amd64]="da782deac10acb1c997b52ea6d0160c06f112a68ad5e25035cfc8e16de0a8bdc" + [arm64]="161223058938c54efadc581b33499210194da58e55cd5ade332e692ca0d649c2" +) + +# The release directory and the package filename both use the bare version, +# while the git tag carries a "v". +dxrt_release="${dxrt_version#v}" +deb_arch=$(dpkg --print-architecture) +deb_file="/tmp/libdxrt-bin_${dxrt_release}_${deb_arch}.deb" + +# Fetch just the one package rather than the source archive: the repository +# keeps every past release under release/, so the tag archive is several +# hundred MB of .deb files we would immediately throw away. Note that +# release/latest is a symlink and would not survive the archive anyway. +wget -qO "${deb_file}" \ + "https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${dxrt_version}/release/${dxrt_release}/libdxrt-bin_${dxrt_release}_${deb_arch}.deb" + +expected_sha256="${dxrt_sha256[${deb_arch}]:-}" +if [[ -z "${expected_sha256}" ]]; then + echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install an unverified package." + exit 1 +fi +actual_sha256=$(sha256sum "${deb_file}" | cut -d' ' -f1) +if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then + echo "SHA-256 mismatch for ${deb_file}:" + echo " expected ${expected_sha256}" + echo " actual ${actual_sha256}" + echo "Refusing to install a package that doesn't match the pinned checksum." + exit 1 +fi + +# Install with dpkg rather than apt. install_deps.sh finishes by purging gnupg +# and clearing /var/lib/apt/lists, so apt here would have to re-fetch about +# 10 MB of package indexes over the network before it could resolve anything, +# which is a build failure waiting to happen for no benefit. The package only +# depends on libc6, libgcc-s1, libncurses6, libstdc++6 and libtinfo6, all of +# which the deps stage already has, and its postinst does nothing but run +# ldconfig and print where the files went. dpkg fails loudly if a dependency +# ever goes missing. +dpkg -i "${deb_file}" + +# The package bundles a dx_engine wheel per cpython ABI tag. Ask dpkg what it +# installed rather than hardcoding a path, so this keeps working if DEEPX moves +# the wheels between releases. +package=$(dpkg-deb -f "${deb_file}" Package) +py_tag="cp$(python3 -c 'import sys; print(f"{sys.version_info[0]}{sys.version_info[1]}")')" +wheel=$(dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" | grep -- "${py_tag}" | sort | head -1 || true) + +if [[ -z "${wheel}" ]]; then + echo "No dx_engine wheel for ${py_tag} in ${package}. Wheels available:" + dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" || echo " (none)" + exit 1 +fi + +pip3 install "${wheel}" + +rm -f "${deb_file}" diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/consumer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/consumer-for new file mode 100644 index 0000000000..c8ae1cc5b9 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/consumer-for @@ -0,0 +1 @@ +dxrtd diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/dependencies.d/log-prepare b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/dependencies.d/log-prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/pipeline-name b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/pipeline-name new file mode 100644 index 0000000000..55265a254e --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/pipeline-name @@ -0,0 +1 @@ +dxrtd-pipeline diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run new file mode 100755 index 0000000000..a02011f742 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run @@ -0,0 +1,4 @@ +#!/command/with-contenv bash +# shellcheck shell=bash + +exec logutil-service /dev/shm/logs/dxrtd diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/type @@ -0,0 +1 @@ +longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/dependencies.d/prepare b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/dependencies.d/prepare new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish new file mode 100755 index 0000000000..8c35e27117 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish @@ -0,0 +1,43 @@ +#!/command/with-contenv bash +# shellcheck shell=bash +# Take down the S6 supervision tree when the service fails + +set -o errexit -o nounset -o pipefail + +# Logs should be sent to stdout so that s6 can collect them + +declare exit_code_container +exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode) +readonly exit_code_container +readonly exit_code_service="${1}" +readonly exit_code_signal="${2}" +readonly service="dxrtd" + +echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})" + +# dxrtd takes itself down when the machine has no DEEPX NPU, which is the +# normal case: this service ships in the default image and most users do not +# have the hardware. That self-shutdown reaches this script as exit 256 / +# signal 15, indistinguishable from an operator stopping the container, so +# without the marker written by run every machine lacking a DEEPX NPU would +# have Frigate halted at startup. +if [[ -f /dev/shm/.dxrtd-no-hardware ]]; then + echo "[INFO] Service ${service} stopped because no DEEPX NPU is present; leaving the rest of Frigate running" + exit 0 +fi + +# Past this point the service really was running, so only a genuine failure +# should bring the container down. +if [[ "${exit_code_service}" -eq 256 ]]; then + if [[ "${exit_code_container}" -eq 0 ]]; then + echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode + fi + if [[ "${exit_code_signal}" -eq 15 ]]; then + exec /run/s6/basedir/bin/halt + fi +elif [[ "${exit_code_service}" -ne 0 ]]; then + if [[ "${exit_code_container}" -eq 0 ]]; then + echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode + fi + exec /run/s6/basedir/bin/halt +fi diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/producer-for b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/producer-for new file mode 100644 index 0000000000..29cfb55297 --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/producer-for @@ -0,0 +1 @@ +dxrtd-log diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run new file mode 100755 index 0000000000..ac224a1c9e --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run @@ -0,0 +1,41 @@ +#!/command/with-contenv bash +# shellcheck shell=bash +# Start the DEEPX DX-RT runtime daemon (dxrtd) + +set -o errexit -o nounset -o pipefail + +# Logs should be sent to stdout so that s6 can collect them +exec 2>&1 + +# The DEEPX kernel driver runs on the Docker host, not in this container, and +# exposes the NPU as /dev/dxrt* device nodes that are passed through. Without +# them there is no NPU to serve, so exit cleanly and let the rest of Frigate +# start normally: this service ships in the default image and most users do +# not have DEEPX hardware. +if ! compgen -G "/dev/dxrt*" > /dev/null; then + echo "[INFO] No DEEPX NPU device found, not starting dxrtd" + # Tell finish that this shutdown is deliberate. Taking the service down + # here arrives at finish as exit 256 / signal 15, which is exactly what an + # operator stopping the container looks like, so the marker is the only + # thing that tells the two apart. + touch /dev/shm/.dxrtd-no-hardware + exec s6-svc -Od . +fi + +# Reaching here means there is hardware to serve, so clear any marker left by +# an earlier run before the device appeared. +rm -f /dev/shm/.dxrtd-no-hardware + +# dxrtd is a system-wide singleton. If the host's own dxrt.service is still +# running it will refuse to start, so surface that up front rather than +# leaving the user to decode the daemon's own error. +if pgrep -x dxrtd > /dev/null; then + echo "[ERROR] Another dxrtd instance is already running" + echo "[ERROR] dxrtd may only run once per system. Stop the host service with:" + echo "[ERROR] sudo systemctl disable --now dxrt.service" + exit 1 +fi + +echo "[INFO] Starting dxrtd for DEEPX NPU device(s): $(echo /dev/dxrt*)" + +exec /usr/local/bin/dxrtd diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/timeout-kill b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/timeout-kill new file mode 100644 index 0000000000..3a05c8b3ee --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/timeout-kill @@ -0,0 +1 @@ +30000 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/type b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/type new file mode 100644 index 0000000000..5883cff0cd --- /dev/null +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/type @@ -0,0 +1 @@ +longrun diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/dxrtd b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/dxrtd new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run index c493e320ee..3904a5e621 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run @@ -4,7 +4,7 @@ set -o errexit -o nounset -o pipefail -dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync) +dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/dxrtd) mkdir -p "${dirs[@]}" chown nobody:nogroup "${dirs[@]}" diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/dxrtd-pipeline b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/dxrtd-pipeline new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker/main/rootfs/labelmap/voc-20.txt b/docker/main/rootfs/labelmap/voc-20.txt new file mode 100644 index 0000000000..d04fc9e7ef --- /dev/null +++ b/docker/main/rootfs/labelmap/voc-20.txt @@ -0,0 +1,20 @@ +0 aeroplane +1 bicycle +2 bird +3 boat +4 bottle +5 bus +6 car +7 cat +8 chair +9 cow +10 diningtable +11 dog +12 horse +13 motorbike +14 person +15 pottedplant +16 sheep +17 sofa +18 train +19 tvmonitor diff --git a/docs/data/object_detectors_models.yaml b/docs/data/object_detectors_models.yaml index 1c5a189e5e..49fe437f0c 100644 --- a/docs/data/object_detectors_models.yaml +++ b/docs/data/object_detectors_models.yaml @@ -1269,3 +1269,126 @@ axengine: input_dtype: int input_pixel_format: bgr labelmap_path: /labelmap/coco-80.txt + +deepx: + title: DEEPX NPU + models: + - key: yolo + label: YOLO (anchor / anchor-free / NMS-in-head) + recommended: true + download: No model is bundled with Frigate. Download a pre-compiled `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point `model.path` at it. Set `model_format` to match the detection-head layout the model was compiled from. + ui: |- + Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings: + + | Field | Value | + | ---------------------------------------- | ------------------------- | + | **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` | + | **Label map for custom object detector** | `/labelmap/coco-80.txt` | + | **Object detection model input width** | `640` | + | **Object detection model input height** | `640` | + | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | + | **Model Input D Type** | `int` (Frigate's default value) | + + The width and height must match the resolution the `.dxnn` file was compiled for. Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden. + + Set `model_type` and `model_format` on the detector itself rather than in the model settings above. `model_type` is `yolo-generic` here; `model_format` is the detection-head layout the model was compiled from (`anchor`, `anchor_free`, or `nms_in_head`), and leaving it at its `auto` default falls back to inferring the layout from the output shape, which cannot tell an anchor-based head from an anchor-free one. + yaml: |- + detectors: + deepx: + type: deepx + + # Which decoder reads the model's output. + model_type: yolo-generic + + # The detection-head layout the .dxnn was compiled from. Selects + # how the output is decoded. Leave as "auto" to infer it from + # the output shape, which cannot tell an anchor-based head from + # an anchor-free one. + model_format: anchor_free + + # Optional. Comma-separated NPU device indices to bind to. + # Defaults to "auto", which binds to every NPU found. + # device_ids: "0" + + # Set to true when the model was compiled with PPU support, which + # moves candidate selection onto the NPU. + # ppu: false + + model: + path: /config/model_cache/deepx/model.dxnn + width: 640 + height: 640 + labelmap_path: /labelmap/coco-80.txt + - key: ssd + label: SSD (MobileNetV1 / MobileNetV2-Lite / VGG-16) + recommended: false + download: No model is bundled with Frigate. Download a pre-compiled SSD `.dxnn` model (MobileNetV1, MobileNetV2-Lite, or VGG-16 backbone) from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point `model.path` at it. These models are trained on Pascal VOC's 20 classes, not COCO, so `labelmap_path` must be set to `/labelmap/voc-20.txt`. + ui: |- + Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings: + + | Field | Value | + | ---------------------------------------- | ------------------------- | + | **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` | + | **Label map for custom object detector** | `/labelmap/voc-20.txt` | + | **Object detection model input width** | `300` | + | **Object detection model input height** | `300` | + | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | + | **Model Input D Type** | `int` (Frigate's default value) | + + Set `model_type` to `ssd` on the detector itself rather than in the model settings above. No `model_format` is needed: DX-COM keeps SSD's softmax and prior-box decode inside the compiled model, so the same decoder reads all three backbones. + yaml: |- + detectors: + deepx: + type: deepx + + # Which decoder reads the model's output. The .dxnn already + # applies the softmax and prior-box decode, so no model_format + # is needed for any of the three backbones. + model_type: ssd + + # Optional. Comma-separated NPU device indices to bind to. + # Defaults to "auto", which binds to every NPU found. + # device_ids: "0" + + model: + path: /config/model_cache/deepx/model.dxnn + width: 300 + height: 300 + labelmap_path: /labelmap/voc-20.txt + - key: damoyolo + label: DAMO-YOLO (TinyNAS-L20T / S / M / L) + recommended: false + download: No model is bundled with Frigate. Download a pre-compiled DAMO-YOLO `.dxnn` model (TinyNAS-L20T, S, M, or L size) from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point `model.path` at it. All four sizes share one detection head, so no `model_format` is needed. + ui: |- + Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings: + + | Field | Value | + | ---------------------------------------- | ------------------------- | + | **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` | + | **Label map for custom object detector** | `/labelmap/coco-80.txt` | + | **Object detection model input width** | `640` | + | **Object detection model input height** | `640` | + | **Model Input Pixel Color Format** | `rgb` (Frigate's default value) | + | **Model Input Tensor Shape** | `nhwc` (Frigate's default value) | + | **Model Input D Type** | `int` (Frigate's default value) | + + Set `model_type` to `damo-yolo` on the detector itself rather than in the model settings above. + yaml: |- + detectors: + deepx: + type: deepx + + # Which decoder reads the model's output. + model_type: damo-yolo + + # Optional. Comma-separated NPU device indices to bind to. + # Defaults to "auto", which binds to every NPU found. + # device_ids: "0" + + model: + path: /config/model_cache/deepx/model.dxnn + width: 640 + height: 640 + labelmap_path: /labelmap/coco-80.txt diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 602ebfcb85..7e848d0fb6 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -24,6 +24,7 @@ Frigate supports multiple different detectors that work on different types of ha - [Coral EdgeTPU](#edge-tpu-detector): The Google Coral EdgeTPU is available in USB, Mini PCIe, and m.2 formats allowing for a wide range of compatibility with devices. - [Hailo](#hailo-8): The Hailo8 and Hailo8L AI Acceleration module is available in m.2 format with a HAT for RPi devices, offering a wide range of compatibility with devices. - [MemryX](#memryx-mx3): The MX3 Acceleration module is available in m.2 format, offering broad compatibility across various platforms. +- [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, offering broad compatibility across various platforms. **AMD** @@ -572,6 +573,132 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht --- +## DEEPX NPU + +This detector is available for use with the DEEPX NPU, both the DX-M1 M.2 module and the DX-M1M on the Sixfab AI HAT+ for the Raspberry Pi 5. The configuration below applies unchanged to either form factor. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com). + +See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and passing the NPU through to the container. + +To configure a DEEPX detector, set the `type` attribute to `deepx` and follow the configuration guide below. + +### Configuration {#configuration-deepx} + + + +Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format. + +The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of object detection models alongside a JSON file describing how each was compiled: YOLO variants (`model_type: yolo-generic`), SSD (`model_type: ssd`), and DAMO-YOLO (`model_type: damo-yolo`). Pick a model, download the `.dxnn`, bind-mount it into the container, and point `model.path` at it. Note which one you picked, since it determines the `model_type` and `model_format` values below. + +`model_type` is set on the detector rather than on the root `model` block. Frigate ignores a `model` block nested under a detector, and the root `model.model_type` defaults to `ssd`, so leaving it unset would silently decode a YOLO model with the SSD decoder and report no detections at all. The detector's own `model_type` defaults to `yolo-generic` and overrides the root value. + +Alternatively, compile your own model with the DX-COM compiler. + +`model.width` and `model.height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden. + +#### Label maps + +YOLO and DAMO-YOLO models from the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. SSD models are trained on Pascal VOC's 20-class label set instead, and need `labelmap_path` set to `/labelmap/voc-20.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type for any of these. + +#### YOLO model formats {#deepx-yolo-model-formats} + +For `model_type: yolo-generic`, DX-COM preserves the detection head of the model it compiles, so the head's tensor layout determines how the output has to be read. Set `model_format` to the layout your `.dxnn` file was compiled with: + +```yaml +detectors: + deepx: + type: deepx + model_format: anchor_free +``` + +Supported values are three decode families: + +| `model_format` | Output layout | PPU support | +| -------------- | -------------- | ----------- | +| `anchor` | Anchor-based, with a separate objectness score | Yes | +| `anchor_free` | Anchor-free, no objectness, boxes already in pixels | Yes | +| `nms_in_head` | NMS runs in the detection head, output is final corner boxes | No | + +Leaving `model_format` at its `auto` default falls back to inferring the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for anchor-free heads. Setting it explicitly is recommended. + +Two mistakes this option prevents: + +- Decoding an anchor-free model with the anchor formula squares the box dimensions, producing boxes hundreds of thousands of pixels wide that clip to the entire frame. +- Decoding an anchor-based model as anchor-free reads the objectness column as a class score, so confidences and class IDs are both wrong. + +`nms_in_head` models run NMS inside the head, so `nms_threshold` does not apply to them. + +#### SSD models + +For `model_type: ssd`, no `model_format` is needed. DX-COM keeps SSD's postprocessing inside the compiled model: the `.dxnn` runs the NPU head into a small CPU subgraph that concatenates the per-feature-map outputs, applies the softmax, and decodes the prior-box regression all the way to corner-form boxes. Only NMS is left for the host, so nothing about the backbone has to be named. This applies to all three ModelZoo backbones (MobileNetV1, MobileNetV2-Lite, VGG16) even though they emit different numbers of boxes (3000 and 8732). + +```yaml +detectors: + deepx: + type: deepx + model_type: ssd + +model: + width: 300 + height: 300 + labelmap_path: /labelmap/voc-20.txt +``` + +#### DAMO-YOLO models + +For `model_type: damo-yolo`, no `model_format` is needed — all four sizes (TinyNAS-L20T, S, M, L) share one detection head, differing only in backbone depth/width. Leave `model_format` at its `auto` default: + +```yaml +detectors: + deepx: + type: deepx + model_type: damo-yolo + +model: + width: 640 + height: 640 + labelmap_path: /labelmap/coco-80.txt +``` + +#### PPU models + +Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. PPU support is currently only available for `model_type: yolo-generic` — there is no confirmed PPU record layout for SSD or DAMO-YOLO yet, and setting `ppu: true` with either is rejected at startup. + +Set `ppu: true` on the detector when using a PPU-compiled yolo-generic model: + +```yaml +detectors: + deepx: + type: deepx + model_format: anchor_free + ppu: true + score_threshold: 0.25 + nms_threshold: 0.45 +``` + +`ppu` and `model_format` are independent: `ppu` says whether candidate selection already happened on the NPU, and `model_format` says how to read the boxes that come back. Both must match how the model was compiled. + +`model_format` is required whenever `ppu: true` is set. A PPU record has the same fixed-width layout for every yolo-generic variant, so unlike the raw output path there is no shape to infer the layout from — leaving it at `auto` is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch. + +DX-COM cannot compile `nms_in_head` models with PPU support, since their heads already produce final detections. Combining the two is rejected at startup. + +#### Multiple NPUs + +`device_ids` defaults to `auto`, which binds the detector to every NPU it can find. To pin a detector to specific devices, set it to a comma-separated list of the indices you want it to use. This allows multiple detectors to be configured against separate modules: + +```yaml +detectors: + deepx_0: + type: deepx + device_ids: "0" + deepx_1: + type: deepx + device_ids: "1,2" +``` + +A yaml list (`device_ids: [0, 1]`) is accepted as well, but the comma-separated string is what the config UI writes. + +--- + ## NVidia TensorRT Detector Nvidia Jetson devices may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection. diff --git a/docs/docs/frigate/hardware.md b/docs/docs/frigate/hardware.md index 1df3bb8368..1180a76ecc 100644 --- a/docs/docs/frigate/hardware.md +++ b/docs/docs/frigate/hardware.md @@ -65,6 +65,11 @@ Frigate supports multiple different detectors that work on different types of ha - [Supports many model architectures](../../configuration/object_detectors#memryx-mx3) - Runs best with tiny, small, or medium-size models +- [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices. + - [Supports YOLO, SSD, and DAMO-YOLO model architectures](../../configuration/object_detectors#deepx-npu) + - Runs best with tiny or small size models + - Runs efficiently on low power hardware + **AMD** - [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection @@ -256,6 +261,24 @@ The MX3 is a pipelined architecture, where the maximum frames per second support Inference speeds may vary depending on the host platform. The above data was measured on an **Intel 13700 CPU**. Platforms like Raspberry Pi, Orange Pi, and other ARM-based SBCs have different levels of processing capability, which may limit total FPS. +### DEEPX NPU + +Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 module, which works on x86 (Intel/AMD) and ARM-based SBCs such as the Raspberry Pi 5, and the **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) for the Raspberry Pi 5. Both use the same driver and runtime, so the configuration is identical for either one. + +DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com). + +The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO, SSD, and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and YOLO models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage. + +The DEEPX kernel driver has to be installed on the Docker host before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps. + +Detailed information is available [in the detector docs](/configuration/object_detectors#deepx-npu). + +| Name | Input Size | Inference Time | +| ------- | ---------- | -------------- | +| YOLOv8s | 640 | ~ 19 ms | + +Inference times were measured on a Raspberry Pi 5. The host platform affects the pre- and post-processing cost rather than the inference itself. + ### Nvidia Jetson Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. It will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets), and will make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector). diff --git a/docs/docs/frigate/installation.md b/docs/docs/frigate/installation.md index 14c85ec4f3..875945df29 100644 --- a/docs/docs/frigate/installation.md +++ b/docs/docs/frigate/installation.md @@ -377,6 +377,89 @@ If you can't use Docker Compose, you can run the container with something simila Finally, configure [hardware object detection](/configuration/object_detectors#memryx-mx3) to complete the setup. +### DEEPX NPU + +The DEEPX NPU is available in two form factors, and Frigate supports both: + +- **DX-M1** in the M.2 2280 form factor (like an NVMe SSD), for x86 (Intel/AMD) PCs, the Raspberry Pi 5, and other ARM SBCs with an exposed PCIe M.2 slot. +- **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart), a HAT+ board that connects to the Raspberry Pi 5 over PCIe Gen 3 x1. + +Both present the NPU through the same PCIe driver and DX-RT runtime, so the setup below and the detector configuration are identical for either one. Nothing needs to change when moving between them. + +DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com). + +#### Versions + +A DEEPX install has three separately versioned pieces, and all three have to agree. The DX-RT runtime is built into the Frigate image, so the two host-side pieces must be brought up to match it: + +| Component | Version | Installed on | Installed by | +| ------------- | -------- | ------------- | -------------------------- | +| DX-RT runtime | `v3.4.2` | Frigate image | Built in, no action needed | +| Kernel driver | `v2.6.0` | Docker host | `user_installation.sh` | +| NPU firmware | `v2.7.4` | The module | Flashed from the host | + +:::warning + +A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check all three versions before anything else. + +::: + +The firmware version cannot be read without the DX-RT runtime installed on the host, so checking or updating it is a manual step done before Frigate is running: + +1. Install the DX-RT runtime on the host by following DEEPX's own instructions. If you are using Sixfab hardware, their [AI HAT+ quickstart](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) covers this. +2. Check the reported firmware version and update the module if it does not match the table above. +3. **Remove or disable the host runtime before starting the Frigate container.** + +:::danger + +Step 3 is not optional when running Frigate in Docker. The DX-RT daemon (`dxrtd`) may only run once per system, and Frigate runs its own copy inside the container. Leaving the host runtime active means two daemons competing for the same NPU, and the container's copy will fail to start. + +This only applies to Docker. If you run Frigate directly on the host, it uses the host's runtime and there is no conflict. + +::: + +#### Installation + +The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate. + +1. Copy or download [this script](https://github.com/blakeblackshear/frigate/blob/dev/docker/deepx/user_installation.sh). +2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh` +3. Run the script with `./user_installation.sh` +4. **Restart your computer** to complete driver installation. + +Confirm the NPU is visible before continuing: + +```bash +ls /dev/dxrt* +``` + +:::note + +The installation script disables the host's `dxrt.service` if it finds one, for the reason described above. If you install or reinstall the DEEPX runtime on the host afterwards, disable it again before starting the container: + +```bash +sudo systemctl disable --now dxrt.service +``` + +::: + +#### Setup + +To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable` + +Next, grant Docker access to the NPU by adding the device to your `docker-compose.yml` file: + +```yaml +devices: + - /dev/dxrt0 +``` + +If you can't use Docker Compose, add `--device /dev/dxrt0` to your `docker run` command. + +#### Configuration + +Finally, configure [hardware object detection](/configuration/object_detectors#deepx-npu) to complete the setup. + ### Rockchip platform Make sure that you use a linux distribution that comes with the rockchip BSP kernel 5.10 or 6.1 and necessary drivers (especially rkvdec2 and rknpu). To check, enter the following commands: diff --git a/frigate/detectors/detector_config.py b/frigate/detectors/detector_config.py index 52d75ff8f7..70a21a4519 100644 --- a/frigate/detectors/detector_config.py +++ b/frigate/detectors/detector_config.py @@ -36,6 +36,7 @@ class InputDTypeEnum(str, Enum): class ModelTypeEnum(str, Enum): + damoyolo = "damo-yolo" dfine = "dfine" rfdetr = "rfdetr" ssd = "ssd" @@ -93,7 +94,7 @@ class ModelConfig(BaseModel): model_type: ModelTypeEnum = Field( default=ModelTypeEnum.ssd, title="Object Detection Model Type", - description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization.", + description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization.", ) _merged_labelmap: dict[int, str] | None = PrivateAttr() _colormap: dict[int, tuple[int, int, int]] = PrivateAttr() diff --git a/frigate/detectors/plugins/deepx.py b/frigate/detectors/plugins/deepx.py new file mode 100644 index 0000000000..3e28450304 --- /dev/null +++ b/frigate/detectors/plugins/deepx.py @@ -0,0 +1,1143 @@ +"""DEEPX NPU detector running compiled .dxnn models via the DX-RT runtime.""" + +import glob +import logging +import os +from enum import Enum +from typing import Literal + +import cv2 +import numpy as np +from pydantic import ConfigDict, Field, field_validator, model_validator + +from frigate.detectors.detection_api import DetectionApi +from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum +from frigate.util.model import post_process_yolo + +logger = logging.getLogger(__name__) + +DETECTOR_KEY = "deepx" + +DXNN_DEVICES_ENV = "DXNN_DEVICES" + +# Stands in for "let Frigate pick the devices". A plain empty string would be +# written back to the config as a deletion, so the no-op needs a real value. +DEVICE_IDS_AUTO = "auto" + + +class DeepxModelTypeEnum(str, Enum): + """The subset of Frigate's model types the DEEPX decoders can read. + + Frigate keeps `model_type` on the root `model` block and ignores a nested + one, so it is not selectable per detector in the config UI, and an + omitted value silently defaults to `ssd`. Repeating the supported values + here as a detector option makes the choice visible in the form; the + detector writes it back onto the merged model config at startup so the + rest of Frigate agrees with the decoder. + """ + + # PPU-compiled models still use yolo-generic; the `ppu` option is what + # selects the decode path, since there is no PPU-specific model type. + yologeneric = ModelTypeEnum.yologeneric.value + ssd = ModelTypeEnum.ssd.value + damoyolo = ModelTypeEnum.damoyolo.value + + +class ModelFormatEnum(str, Enum): + """How a .dxnn file's output is decoded. + + DX-COM preserves the detection head of the source model, so the exact + layout has to be told apart to read it. Values are grouped by which + `model_type` they apply to; a model_validator on DeepxDetectorConfig + enforces that a given value is only used with its matching type. + """ + + # "no explicit layout", and the default. model_type=yolo-generic infers + # the layout from the output shape; model_type=damo-yolo takes no layout + # at all. This is a real enum member rather than None so the config form + # always round-trips a value -- a null is written back as a deletion. + auto = "auto" + # model_type=yolo-generic: which detection-head tensor layout the .dxnn + # was compiled with. + anchor = "anchor" + anchor_free = "anchor_free" + nms_in_head = "nms_in_head" + + +# Anchor-based heads. Raw output is (N, 5+C) with a separate objectness column, +# and the box columns hold sigmoid activations that must be rescaled against the +# grid cell and anchor that produced them. +ANCHOR_FORMATS = frozenset( + { + ModelFormatEnum.anchor, + } +) + +# Anchor-free heads. Raw output is (4+C, N) channel-major with no objectness +# column, and the box columns are already in input pixels. +ANCHOR_FREE_FORMATS = frozenset( + { + ModelFormatEnum.anchor_free, + } +) + +# Heads that run NMS on-device and emit final boxes as (N, 6) corner records. +# DX-COM cannot compile these with PPU support. +NMS_IN_HEAD_FORMATS = frozenset( + { + ModelFormatEnum.nms_in_head, + } +) + +# Anchor table for anchor-based PPU models, keyed by stride. +PPU_ANCHORS = { + 8: np.array([[10, 13], [16, 30], [33, 23]], dtype=np.float32), + 16: np.array([[30, 61], [62, 45], [59, 119]], dtype=np.float32), + 32: np.array([[116, 90], [156, 198], [373, 326]], dtype=np.float32), +} +PPU_STRIDES = np.array([8, 16, 32], dtype=np.float32) + +# Byte layout of the fixed-width detection record the PPU emits. +PPU_RECORD_SIZE = 32 +PPU_BOX_BYTES = (0, 16) +PPU_GRID_BYTES = (16, 20) +PPU_SCORE_BYTES = (20, 24) +PPU_LABEL_BYTES = (24, 28) + + +# DAMO-YOLO's ZeroHead is anchor-free at these three strides regardless of +# size (TinyNAS-L20T/S/M/L differ only in backbone depth/width). +DAMOYOLO_STRIDES = (8, 16, 32) + + +def detect_device_count() -> int: + """Count the NPU devices visible to this process. + + Falls back from the DX-RT API to a scan of the passed-through device + nodes, then to a single device. + """ + try: + from dx_engine import get_device_count + + count = get_device_count() + except (ImportError, RuntimeError) as err: + logger.debug("Could not query DX-RT for a device count: %s", err) + else: + if count > 0: + return count + + devices = glob.glob("/dev/dxrt*") + + return len(devices) if devices else 1 + + +def parse_device_ids(value: str) -> list[int]: + """Parse a comma-separated list of NPU device indices. + + The `auto` sentinel and an empty value both mean "nothing was picked", + which leaves device selection to `resolve_devices`. + """ + if value.strip().lower() == DEVICE_IDS_AUTO: + return [] + + return [int(part) for part in value.split(",") if part.strip()] + + +def resolve_devices(configured: str) -> list[int]: + """Resolve which NPU device indices this detector should bind to.""" + if configured: + # already validated by DeepxDetectorConfig + devices = parse_device_ids(configured) + + if devices: + return devices + + env_value = os.environ.get(DXNN_DEVICES_ENV) + + if env_value: + try: + devices = parse_device_ids(env_value) + except ValueError: + logger.warning( + "Ignoring unparsable %s=%s, detecting devices instead", + DXNN_DEVICES_ENV, + env_value, + ) + else: + if devices: + return devices + + return list(range(detect_device_count())) + + +def reinterpret(tensor: np.ndarray, byte_range: tuple[int, int], dtype) -> np.ndarray: + """Read a byte column range of a uint8 PPU tensor as `dtype`. + + A column slice is not contiguous, and numpy will not reinterpret + non-contiguous memory as a differently sized dtype, so it is copied first. + """ + lo, hi = byte_range + return np.ascontiguousarray(tensor[:, lo:hi]).view(dtype) + + +def ppu_records(outputs: list[np.ndarray]) -> np.ndarray | None: + """Return the fixed-width record array a PPU model emits, or None.""" + if not outputs or outputs[0].ndim < 2: + return None + + # drop the batch dimension, leaving one uint8 record per candidate + records = outputs[0][0] + + if ( + records.ndim != 2 + or records.shape[1] != PPU_RECORD_SIZE + or records.shape[0] == 0 + ): + logger.debug("Unexpected PPU output shape %s, skipping frame", records.shape) + return None + + return records + + +def _softmax(x: np.ndarray) -> np.ndarray: + """Numerically stable softmax over the last axis.""" + shifted = x - np.max(x, axis=-1, keepdims=True) + exp = np.exp(shifted) + return exp / np.sum(exp, axis=-1, keepdims=True) + + +def _split_box_and_class_outputs( + outputs: list[np.ndarray], +) -> tuple[np.ndarray, np.ndarray] | None: + """Split a 2-tensor raw output into (box tensor, class tensor) by shape. + + Used for model types where DX-COM's raw output tensor order isn't + confirmed. Whichever tensor's shape reads as box-like -- (..., 4), or + (..., 4, bins) for a per-side distribution -- is treated as boxes; the + other is class scores. Returns None if that can't be told apart, which + a class-score tensor with exactly 4 classes would also trigger. + """ + if len(outputs) != 2: + return None + + def is_box_shaped(t: np.ndarray) -> bool: + return (t.ndim == 3 and t.shape[-1] == 4) or (t.ndim == 4 and t.shape[-2] == 4) + + first, second = outputs[0], outputs[1] + first_is_box, second_is_box = is_box_shaped(first), is_box_shaped(second) + + if first_is_box and not second_is_box: + return first, second + if second_is_box and not first_is_box: + return second, first + + return None + + +def _shapes(outputs: list[np.ndarray]) -> list[tuple[int, ...]]: + """Output shapes, for diagnosing a decode path that can't read them.""" + return [np.shape(output) for output in outputs] + + +def fill_detections( + x_min: np.ndarray, + y_min: np.ndarray, + x_max: np.ndarray, + y_max: np.ndarray, + scores: np.ndarray, + labels: np.ndarray, + width: int, + height: int, + order: np.ndarray, +) -> np.ndarray: + """Normalize the surviving boxes into Frigate's (20, 6) detection array.""" + detections = np.zeros((20, 6), np.float32) + + for i, idx in enumerate(order[:20]): + detections[i] = [ + labels[idx], + scores[idx], + np.clip(y_min[idx] / height, 0, 1), + np.clip(x_min[idx] / width, 0, 1), + np.clip(y_max[idx] / height, 0, 1), + np.clip(x_max[idx] / width, 0, 1), + ] + + return detections + + +def run_nms( + x_min: np.ndarray, + y_min: np.ndarray, + box_w: np.ndarray, + box_h: np.ndarray, + scores: np.ndarray, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Return the indices surviving NMS, in descending score order.""" + if logger.isEnabledFor(logging.DEBUG): + # Separates "the decode produced nothing" from "it produced boxes + # whose scores all fell under the threshold", which look identical + # from the empty detection array alone. + logger.debug( + "NMS input: %d boxes, best score %.4f, %d over threshold %.2f", + len(scores), + float(scores.max()) if len(scores) else 0.0, + int((scores >= score_threshold).sum()), + score_threshold, + ) + + boxes_xywh = np.column_stack([x_min, y_min, box_w, box_h]) + indices = cv2.dnn.NMSBoxes( + boxes_xywh.tolist(), scores.tolist(), score_threshold, nms_threshold + ) + + if len(indices) == 0: + return np.empty(0, dtype=np.int32) + + return np.array(indices).reshape(-1) + + +def decode_ppu_anchor( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode PPU output from an anchor-based head. + + Models compiled with DEEPX's Post-Processing Unit perform candidate + selection on the NPU and emit one fixed-width record per surviving + detection, so only the anchor decode and NMS are left for the host. This + is cheaper than decoding raw feature maps because every sub-threshold + anchor has already been discarded. + + Returns detections in Frigate's (20, 6) format of + [class_id, score, y_min, x_min, y_max, x_max] with normalized coordinates. + """ + records = ppu_records(outputs) + + if records is None: + return np.zeros((20, 6), np.float32) + + boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) + scores = reinterpret(records, PPU_SCORE_BYTES, np.float32).flatten() + labels = reinterpret(records, PPU_LABEL_BYTES, np.uint32).flatten() + + grid = records[:, PPU_GRID_BYTES[0] : PPU_GRID_BYTES[1]] + grid_y = grid[:, 0].astype(np.float32) + grid_x = grid[:, 1].astype(np.float32) + anchor_idx = grid[:, 2] + layer_idx = grid[:, 3] + + stride = PPU_STRIDES[layer_idx] + anchor_w = np.zeros(len(boxes), dtype=np.float32) + anchor_h = np.zeros(len(boxes), dtype=np.float32) + + for layer_stride, anchors in PPU_ANCHORS.items(): + mask = stride == layer_stride + + if np.any(mask): + anchor_w[mask] = anchors[anchor_idx[mask], 0] + anchor_h[mask] = anchors[anchor_idx[mask], 1] + + # Anchor-based decode: the model emits sigmoid outputs that are rescaled + # against the grid cell and its anchor to recover pixel geometry. + center_x = (boxes[:, 0] * 2.0 - 0.5 + grid_x) * stride + center_y = (boxes[:, 1] * 2.0 - 0.5 + grid_y) * stride + box_w = (boxes[:, 2] ** 2 * 4.0) * anchor_w + box_h = (boxes[:, 3] ** 2 * 4.0) * anchor_h + + x_min = center_x - box_w * 0.5 + y_min = center_y - box_h * 0.5 + + order = run_nms(x_min, y_min, box_w, box_h, scores, score_threshold, nms_threshold) + + return fill_detections( + x_min, + y_min, + x_min + box_w, + y_min + box_h, + scores, + labels, + width, + height, + order, + ) + + +def decode_ppu_anchor_free( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode PPU output from an anchor-free head. + + These heads have no anchors and no objectness, so the PPU writes the box + directly as (center_x, center_y, width, height) in input pixels. The grid + columns an anchor-based record carries are unused here, and applying the + anchor formula to them is what produces boxes hundreds of thousands of + pixels wide. + """ + records = ppu_records(outputs) + + if records is None: + return np.zeros((20, 6), np.float32) + + boxes = reinterpret(records, PPU_BOX_BYTES, np.float32).reshape(-1, 4) + scores = reinterpret(records, PPU_SCORE_BYTES, np.float32).flatten() + labels = reinterpret(records, PPU_LABEL_BYTES, np.uint32).flatten() + + box_w = boxes[:, 2] + box_h = boxes[:, 3] + x_min = boxes[:, 0] - box_w * 0.5 + y_min = boxes[:, 1] - box_h * 0.5 + + order = run_nms(x_min, y_min, box_w, box_h, scores, score_threshold, nms_threshold) + + return fill_detections( + x_min, + y_min, + x_min + box_w, + y_min + box_h, + scores, + labels, + width, + height, + order, + ) + + +def decode_raw_anchor( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode raw output from an anchor-based head. + + The exported head emits (N, 5+C) rows of + [center_x, center_y, width, height, objectness, class scores...] already in + input pixels. Confidence is objectness times the best class score, which is + what separates this layout from the anchor-free one: reading column 4 as a + class score there would treat objectness as a class. + """ + tensor = outputs[0] + + if tensor.ndim < 2 or tensor.shape[-1] == 0: + return np.zeros((20, 6), np.float32) + + # squeeze would collapse the row axis too when a single candidate is + # returned, so fold every leading axis into rows instead + predictions = tensor.reshape(-1, tensor.shape[-1]) + + if predictions.shape[0] == 0: + return np.zeros((20, 6), np.float32) + + objectness = predictions[:, 4] + class_scores = predictions[:, 5:] + labels = np.argmax(class_scores, axis=1) + scores = objectness * class_scores[np.arange(len(labels)), labels] + + box_w = predictions[:, 2] + box_h = predictions[:, 3] + x_min = predictions[:, 0] - box_w * 0.5 + y_min = predictions[:, 1] - box_h * 0.5 + + order = run_nms(x_min, y_min, box_w, box_h, scores, score_threshold, nms_threshold) + + return fill_detections( + x_min, + y_min, + x_min + box_w, + y_min + box_h, + scores, + labels, + width, + height, + order, + ) + + +def decode_raw_anchor_free( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode raw output from an anchor-free head, or an unconfirmed layout. + + The exported head emits a single tensor with no objectness column, + (4+C, N) channel-major or already transposed to (N, 4+C), and box columns + already in input pixels as (center_x, center_y, width, height). This is + also what `model_format: auto` falls back to: with only one output + tensor there is no objectness column to tell an anchor-based head from + an anchor-free one apart, so every column past the box is read as a + class score, which is only correct for an anchor-free head -- matching + `auto` being "only reliable for anchor-free heads" per its config + description. + """ + predictions = outputs[0] + + # Drop the batch axis without a blind squeeze, which would also collapse + # the box-count axis on the single-candidate frame case. Fold every + # leading axis into rows instead. + if predictions.ndim == 3 and predictions.shape[0] == 1: + predictions = predictions[0] + + if predictions.ndim != 2: + return np.zeros((20, 6), np.float32) + + # DX-COM may export channel-major (4+C, N); transpose to (N, 4+C) so + # rows are boxes, matching every other decoder in this file. + if predictions.shape[0] < predictions.shape[1]: + predictions = predictions.T + + if predictions.shape[0] == 0 or predictions.shape[1] <= 4: + return np.zeros((20, 6), np.float32) + + class_scores = predictions[:, 4:] + labels = np.argmax(class_scores, axis=1) + scores = class_scores[np.arange(len(labels)), labels] + + box_w = predictions[:, 2] + box_h = predictions[:, 3] + x_min = predictions[:, 0] - box_w * 0.5 + y_min = predictions[:, 1] - box_h * 0.5 + + order = run_nms(x_min, y_min, box_w, box_h, scores, score_threshold, nms_threshold) + + return fill_detections( + x_min, + y_min, + x_min + box_w, + y_min + box_h, + scores, + labels, + width, + height, + order, + ) + + +def decode_raw_nms_in_head( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, +) -> np.ndarray: + """Decode raw output from a head that already ran NMS. + + These heads emit (N, 6) rows of [x_min, y_min, x_max, y_max, score, class] + in input pixels, sorted by score. Running NMS again would be wasted work, + so only the score filter and normalization are applied. + """ + tensor = outputs[0] + + if tensor.ndim < 2 or tensor.shape[-1] == 0: + return np.zeros((20, 6), np.float32) + + # squeeze would collapse the row axis too when a single candidate is + # returned, so fold every leading axis into rows instead + predictions = tensor.reshape(-1, tensor.shape[-1]) + + if predictions.shape[0] == 0: + return np.zeros((20, 6), np.float32) + + scores = predictions[:, 4] + order = np.flatnonzero(scores >= score_threshold) + + return fill_detections( + predictions[:, 0], + predictions[:, 1], + predictions[:, 2], + predictions[:, 3], + scores, + predictions[:, 5].astype(np.int32), + width, + height, + order, + ) + + +def decode_ssd_raw( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, +) -> np.ndarray: + """Decode SSD output: already-decoded boxes plus per-class probabilities. + + DX-COM keeps SSD's postprocessing inside the compiled model. The .dxnn + runs the NPU head into a small CPU subgraph that concatenates the + per-feature-map outputs, applies softmax to the confidences, and decodes + the prior-box regression (center variance 0.1, size variance 0.2, prior + table baked in as initializers) all the way to corner-form boxes. That + leaves only NMS for the host, which is what DEEPX's own ssdmv1, + ssdmv2lite and ssdvgg16 demos do -- their postprocessor has no prior + table, no variance constant, and no softmax. + + Expects two output tensors, order-independent: boxes shaped (1, N, 4) as + (x_min, y_min, x_max, y_max) normalized to [0, 1], and probabilities + shaped (1, N, num_classes + 1) with background at class 0. Background is + dropped and surviving labels are 0-indexed, to match a labelmap with no + background row. N is whatever the backbone produces (3000 for the + MobileNet variants, 8732 for VGG16) and never has to be known ahead of + time. + """ + split = _split_box_and_class_outputs(outputs) + + if split is None: + logger.debug( + "Could not identify SSD box/class outputs in %d tensor(s) with " + "shapes %s, skipping frame", + len(outputs), + _shapes(outputs), + ) + return np.zeros((20, 6), np.float32) + + boxes, confidences = split + + if ( + boxes.ndim != 3 + or confidences.ndim != 3 + or boxes.shape[0] == 0 + or boxes.shape[1] == 0 + ): + logger.debug( + "Unexpected SSD output rank: boxes %s, classes %s, skipping frame", + boxes.shape, + confidences.shape, + ) + return np.zeros((20, 6), np.float32) + + boxes = boxes[0] + confidences = confidences[0] + + if boxes.shape[0] != confidences.shape[0]: + logger.debug( + "SSD box count %d does not match class count %d, skipping frame", + boxes.shape[0], + confidences.shape[0], + ) + return np.zeros((20, 6), np.float32) + + # the model already softmaxed these; drop background (class 0) and + # relabel 0-indexed to match a labelmap with no background row + scores = confidences[:, 1:] + labels = np.argmax(scores, axis=1) + scores = scores[np.arange(len(labels)), labels] + + # normalized corner form -> input pixels, which is what NMS and + # fill_detections work in + x_min = boxes[:, 0] * width + y_min = boxes[:, 1] * height + x_max = boxes[:, 2] * width + y_max = boxes[:, 3] * height + + order = run_nms( + x_min, + y_min, + x_max - x_min, + y_max - y_min, + scores, + score_threshold, + nms_threshold, + ) + + return fill_detections( + x_min, y_min, x_max, y_max, scores, labels, width, height, order + ) + + +def _dfl_integral(distribution: np.ndarray) -> np.ndarray: + """Reduce a DFL distribution to an expected distance via weighted sum. + + Ported from tinyvision/DAMO-YOLO's Integral module (Apache-2.0): each of + the 4 box sides is predicted as a probability distribution over + `reg_max + 1` discrete distance bins rather than a single regressed + value; this recovers one scalar distance per side, in stride units. + """ + reg_max = distribution.shape[-1] - 1 + project = np.arange(reg_max + 1, dtype=np.float32) + return distribution @ project + + +def _damoyolo_center_priors( + width: int, height: int, strides: tuple[int, ...] +) -> np.ndarray: + """One (center_x, center_y, stride) row per grid cell across all scales, + in pixel space at the model's own input resolution. + + Ported from tinyvision/DAMO-YOLO's + ZeroHead.get_single_level_center_priors (Apache-2.0). + """ + rows = [] + + for stride in strides: + grid_h, grid_w = height // stride, width // stride + row_idx, col_idx = np.mgrid[0:grid_h, 0:grid_w] + centers_x = col_idx.ravel().astype(np.float32) * stride + centers_y = row_idx.ravel().astype(np.float32) * stride + strides_col = np.full(centers_x.shape, stride, dtype=np.float32) + rows.append(np.stack([centers_x, centers_y, strides_col], axis=1)) + + return np.concatenate(rows, axis=0) + + +def decode_damoyolo_raw( + outputs: list[np.ndarray], + width: int, + height: int, + score_threshold: float, + nms_threshold: float, + strides: tuple[int, ...] = DAMOYOLO_STRIDES, +) -> np.ndarray: + """Decode DAMO-YOLO's raw output: sigmoid class scores + a DFL box head. + + Ported from tinyvision/DAMO-YOLO's ZeroHead (Apache-2.0). DAMO-YOLO is + anchor-free like YOLOv8 and newer, but unlike them its box regression is + a per-side Distribution Focal Loss histogram rather than a value already + in pixels, so it needs its own decode instead of reusing + decode_raw_anchor. Not yet verified against a real compiled model -- + see the two shape branches below for what DX-COM's export is assumed to + produce, and adjust if a real .dxnn's shapes don't match either one. + + Expects two output tensors, order-independent: class scores shaped + (1, N, num_classes), already sigmoid-activated, and a box output that is + either (1, N, 4, reg_max + 1) -- a raw DFL distribution per side, decoded + here via softmax + integral + stride-scaled distance-to-box -- or + already (1, N, 4), in case DX-COM's export folds that whole decode in + (the reference only ever produces a plain 4-wide box tensor as the + *result* of that decode, so this shape is treated as final + (x_min, y_min, x_max, y_max) pixel coordinates, not as a second + distance representation). + """ + split = _split_box_and_class_outputs(outputs) + + if split is None: + logger.debug( + "Could not identify DAMO-YOLO box/class outputs in %d tensor(s) with " + "shapes %s, skipping frame", + len(outputs), + _shapes(outputs), + ) + return np.zeros((20, 6), np.float32) + + box_output, cls_scores = split + + if cls_scores.ndim != 3 or cls_scores.shape[0] == 0 or cls_scores.shape[1] == 0: + return np.zeros((20, 6), np.float32) + + cls_scores = cls_scores[0] + num_priors = cls_scores.shape[0] + + if box_output.ndim == 4: + # raw per-side DFL distribution: softmax the bins, reduce to an + # expected distance (in stride units), then scale by each prior's + # own stride and apply it against that prior's center + distances = _dfl_integral(_softmax(box_output[0])) + + if distances.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO box count %d does not match class count %d, skipping frame", + distances.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + priors = _damoyolo_center_priors(width, height, strides) + + if priors.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO prior count %d does not match model output %d, " + "skipping frame", + priors.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + center_x, center_y, stride = priors[:, 0], priors[:, 1], priors[:, 2] + distances_px = distances * stride[:, None] + + x_min = center_x - distances_px[:, 0] + y_min = center_y - distances_px[:, 1] + x_max = center_x + distances_px[:, 2] + y_max = center_y + distances_px[:, 3] + elif box_output.ndim == 3 and box_output.shape[-1] == 4: + boxes = box_output[0] + + if boxes.shape[0] != num_priors: + logger.debug( + "DAMO-YOLO box count %d does not match class count %d, skipping frame", + boxes.shape[0], + num_priors, + ) + return np.zeros((20, 6), np.float32) + + x_min, y_min, x_max, y_max = ( + boxes[:, 0], + boxes[:, 1], + boxes[:, 2], + boxes[:, 3], + ) + else: + logger.debug( + "Unexpected DAMO-YOLO box output shape %s, skipping frame", + box_output.shape, + ) + return np.zeros((20, 6), np.float32) + + labels = np.argmax(cls_scores, axis=1) + scores = cls_scores[np.arange(num_priors), labels] + + order = run_nms( + x_min, + y_min, + x_max - x_min, + y_max - y_min, + scores, + score_threshold, + nms_threshold, + ) + + return fill_detections( + x_min, y_min, x_max, y_max, scores, labels, width, height, order + ) + + +class DeepxDetectorConfig(BaseDetectorConfig): + """DEEPX NPU detector running .dxnn models via the DX-RT runtime.""" + + model_config = ConfigDict(title="DEEPX NPU") + + type: Literal[DETECTOR_KEY] + device_ids: str = Field( + default=DEVICE_IDS_AUTO, + title="DEEPX device indices", + description="Comma-separated NPU device indices this detector should " + 'bind to, e.g. "0,1". Leave as "auto" to detect them from the ' + "DXNN_DEVICES environment variable, then the DX-RT device count, then " + "a scan of /dev/dxrt*.", + ) + model_type: DeepxModelTypeEnum = Field( + default=DeepxModelTypeEnum.yologeneric, + title="Detection model architecture", + description="Which decoder reads the model's output. Overrides " + "model.model_type for this detector, whose default of ssd would " + "otherwise apply whenever the root model block leaves it unset.", + ) + ppu: bool = Field( + default=False, + title="PPU-compiled model", + description="Set to true when the model was compiled with DEEPX's " + "Post-Processing Unit support, which moves candidate selection onto " + "the NPU and changes the output format. Only supported for " + "model_type yolo-generic.", + ) + model_format: ModelFormatEnum = Field( + default=ModelFormatEnum.auto, + title="Detection head layout the model was compiled from", + description="Selects how the model's output is decoded. Only used " + 'for model_type yolo-generic: leave as "auto" to infer the layout ' + "from the output shape, which cannot tell an anchor-based head from " + "an anchor-free one and is only reliable for anchor-free heads. " + "Required when ppu is true, since a PPU record's layout is identical " + "for every yolo-generic variant and cannot be inferred. ssd and " + "damo-yolo each read one fixed layout and take no value here.", + ) + score_threshold: float = Field( + default=0.25, + title="Score threshold used when decoding detector output", + ) + nms_threshold: float = Field( + default=0.45, + title="NMS IoU threshold used when decoding detector output", + ) + + @field_validator("device_ids", mode="before") + @classmethod + def coerce_device_ids(cls, value): + """Accept the yaml list form as well as the comma-separated string + the config form writes, since a bare list is the natural thing to + hand-write. A blank entry folds back to the auto sentinel.""" + if isinstance(value, (list, tuple)): + value = ",".join(str(item) for item in value) + elif isinstance(value, int) and not isinstance(value, bool): + value = str(value) + + if value is None or (isinstance(value, str) and not value.strip()): + return DEVICE_IDS_AUTO + + return value + + @field_validator("device_ids") + @classmethod + def validate_device_ids(cls, value: str) -> str: + """Reject a device list that can't be parsed, so the error surfaces at + config load rather than when the detector process starts.""" + try: + parse_device_ids(value) + except ValueError: + raise ValueError( + f'device_ids "{value}" is not a comma-separated list of NPU ' + f'device indices, e.g. "0,1", or "{DEVICE_IDS_AUTO}".' + ) from None + + return value + + @field_validator("model_format", mode="before") + @classmethod + def coerce_model_format(cls, value): + """A blank or absent entry means "no explicit layout".""" + if value is None or (isinstance(value, str) and not value.strip()): + return ModelFormatEnum.auto + + return value + + @model_validator(mode="after") + def validate_ppu_is_supported(self): + """Reject a PPU pairing DX-COM cannot produce, or that has no + confirmed PPU record layout yet.""" + if not self.ppu: + return self + + if self.model_format in NMS_IN_HEAD_FORMATS: + raise ValueError( + f"model_format '{self.model_format.value}' runs NMS in the " + "detection head and cannot be compiled with PPU support by " + "DX-COM. Set ppu to false." + ) + + if self.model_type is DeepxModelTypeEnum.damoyolo: + raise ValueError( + 'PPU is not yet supported for model_type "damo-yolo". Set ppu to false.' + ) + + return self + + @model_validator(mode="after") + def validate_ppu_requires_model_format(self): + """Reject an unset model_format when ppu is enabled. + + A PPU record is the same fixed-width layout for every yolo-generic + variant, so unlike the raw output path there is no shape to infer + the layout from. Leaving model_format unset here used to silently + default to the anchor-based decoder, which corrupts detections from + anchor-free PPU models instead of failing loudly. + """ + if self.ppu and self.model_format is ModelFormatEnum.auto: + raise ValueError( + "model_format must be set when ppu is true. The PPU output " + "layout can't be inferred, and guessing wrong silently " + "corrupts detections." + ) + + return self + + @model_validator(mode="after") + def validate_model_format_matches_model_type(self): + """Reject a model_format on a model_type that has no layout to pick. + + Only yolo-generic has variants the decoder can't tell apart. SSD and + DAMO-YOLO each read one fixed output layout, so any value other than + the sentinel is a config mistake rather than a choice.""" + if self.model_type is DeepxModelTypeEnum.yologeneric: + return self + + if self.model_format is not ModelFormatEnum.auto: + raise ValueError( + "model_format is not used for model_type " + f"{self.model_type.value} -- its output layout is fixed. " + 'Leave it as "auto".' + ) + + return self + + +class Deepx(DetectionApi): + type_key = DETECTOR_KEY + + def __init__(self, config: DeepxDetectorConfig): + try: + from dx_engine import InferenceEngine, InferenceOption + except ImportError as err: + raise ImportError( + "The DX-RT runtime is not available. Check that the DEEPX " + "kernel driver is loaded on the Docker host and that " + "/dev/dxrt* is passed through to this container." + ) from err + + super().__init__(config) + + # Frigate ignores a nested `model` block, so model.model_type carries + # the root value -- `ssd` whenever it was left unset. The detector's + # own model_type is the one the user actually picked, so it wins, and + # is written back for everything downstream that reads the model + # config (the events table, /api/config). + self.model_type = ModelTypeEnum(config.model_type.value) + config.model.model_type = self.model_type + + model_path = config.model.path + + if not model_path or not os.path.isfile(model_path): + raise FileNotFoundError( + f"DEEPX model '{model_path}' was not found. Compile a model with " + "DX-COM or download one from the DEEPX ModelZoo, then point " + "model.path at the .dxnn file." + ) + + devices = resolve_devices(config.device_ids) + logger.info("Loading DEEPX model %s on device(s) %s", model_path, devices) + + options = InferenceOption() + options.devices = devices + options.bound_option = InferenceOption.BOUND_OPTION.NPU_ALL + + self.session = InferenceEngine(str(model_path), options) + self.ppu = config.ppu + self.model_format = config.model_format + self.score_threshold = config.score_threshold + self.nms_threshold = config.nms_threshold + + self.logged_layout = False + + logger.info( + "DEEPX decoding %s output for model_type %s as %s", + "PPU" if self.ppu else "raw", + self.model_type.value, + "auto-detected" + if self.model_format is ModelFormatEnum.auto + else self.model_format.value, + ) + + def decode(self, outputs: list[np.ndarray]) -> np.ndarray: + """Decode raw or PPU output according to the configured model type + and detection head layout.""" + if self.model_type == ModelTypeEnum.ssd: + return decode_ssd_raw( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + if self.model_type == ModelTypeEnum.damoyolo: + return decode_damoyolo_raw( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + # model_type is yolo-generic from here on + if self.ppu: + # Config validation guarantees model_format is set to an anchor + # or anchor-free format whenever ppu is true, since PPU records + # give no other way to tell the two apart. + if self.model_format in ANCHOR_FREE_FORMATS: + return decode_ppu_anchor_free( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + return decode_ppu_anchor( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + if self.model_format in NMS_IN_HEAD_FORMATS: + return decode_raw_nms_in_head( + outputs, self.width, self.height, self.score_threshold + ) + + # Only single-output anchor heads need the dedicated decoder. A + # multi-part output is a set of raw feature maps, which the shared + # helper already decodes with the same anchor formula. + if self.model_format in ANCHOR_FORMATS and len(outputs) == 1: + return decode_raw_anchor( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + # `auto` (the default) and an explicit `anchor_free` both read a + # single tensor with no objectness column. Routing these through the + # dedicated decoder keeps them honoring this detector's own + # score_threshold and nms_threshold + if len(outputs) == 1 and ( + self.model_format in ANCHOR_FREE_FORMATS + or self.model_format is ModelFormatEnum.auto + ): + return decode_raw_anchor_free( + outputs, + self.width, + self.height, + self.score_threshold, + self.nms_threshold, + ) + + return post_process_yolo(outputs, self.width, self.height) + + def detect_raw(self, tensor_input): + """Run inference and decode the DEEPX model output.""" + outputs = self.session.run([tensor_input]) + + if not isinstance(outputs, list): + outputs = [outputs] + + detections = self.decode(outputs) + + if not self.logged_layout: + self.logged_layout = True + self.log_layout(tensor_input, outputs) + + if logger.isEnabledFor(logging.DEBUG): + logger.debug( + "DEEPX decoded %d detections, best score %.4f", + int((detections[:, 1] > 0).sum()), + float(detections[:, 1].max()), + ) + + return detections + + def log_layout(self, tensor_input, outputs: list[np.ndarray]) -> None: + """Log the model's real input and output layout, once per start. + + Every decode path is a guess about the layout DX-COM compiled the + detection head into, and a wrong guess returns an empty detection + array rather than failing. These shapes and value ranges are the + first thing needed to tell those apart, so they go out at info + level: waiting to reproduce them with debug logging enabled costs a + restart, and this is one line per tensor per detector start. + """ + tensors = [("input", tensor_input)] + tensors += [(f"output[{i}]", out) for i, out in enumerate(outputs)] + + for name, tensor in tensors: + array = np.asarray(tensor) + logger.info( + "DEEPX %s: shape=%s dtype=%s min=%s max=%s", + name, + array.shape, + array.dtype, + array.min() if array.size else "n/a", + array.max() if array.size else "n/a", + ) diff --git a/frigate/test/test_deepx.py b/frigate/test/test_deepx.py new file mode 100644 index 0000000000..00ee72ae44 --- /dev/null +++ b/frigate/test/test_deepx.py @@ -0,0 +1,757 @@ +"""Tests for the DEEPX detector's per-format output decoding.""" + +import unittest + +import numpy as np +from pydantic import ValidationError + +from frigate.detectors.detector_config import ModelTypeEnum +from frigate.detectors.plugins.deepx import ( + ANCHOR_FORMATS, + ANCHOR_FREE_FORMATS, + DEVICE_IDS_AUTO, + NMS_IN_HEAD_FORMATS, + PPU_ANCHORS, + PPU_RECORD_SIZE, + DeepxDetectorConfig, + DeepxModelTypeEnum, + ModelFormatEnum, + decode_damoyolo_raw, + decode_ppu_anchor, + decode_ppu_anchor_free, + decode_raw_anchor, + decode_raw_anchor_free, + decode_raw_nms_in_head, + decode_ssd_raw, + parse_device_ids, + resolve_devices, +) + + +def build_ppu_record( + box: tuple[float, float, float, float], + grid_y: int, + grid_x: int, + anchor_idx: int, + layer_idx: int, + score: float, + label: int, +) -> np.ndarray: + """Build a single fixed-width detection record as the PPU emits it.""" + record = np.zeros(PPU_RECORD_SIZE, dtype=np.uint8) + record[0:16] = np.array(box, dtype=np.float32).view(np.uint8) + record[16:20] = [grid_y, grid_x, anchor_idx, layer_idx] + record[20:24] = np.array([score], dtype=np.float32).view(np.uint8) + record[24:28] = np.array([label], dtype=np.uint32).view(np.uint8) + return record + + +class TestDeepxPpuDecode(unittest.TestCase): + def test_decodes_a_single_detection(self): + # stride 8 (layer 0) with the first anchor of that layer + outputs = [ + build_ppu_record( + box=(0.5, 0.5, 0.5, 0.5), + grid_y=10, + grid_x=10, + anchor_idx=0, + layer_idx=0, + score=0.9, + label=2, + ).reshape(1, 1, PPU_RECORD_SIZE) + ] + + detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + + anchor_w, anchor_h = PPU_ANCHORS[8][0] + # center = (0.5 * 2 - 0.5 + 10) * 8, size = (0.5 ** 2 * 4) * anchor + center = (0.5 * 2.0 - 0.5 + 10) * 8 + box_w = 0.5**2 * 4.0 * anchor_w + box_h = 0.5**2 * 4.0 * anchor_h + + self.assertEqual(detections[0][0], 2) + self.assertAlmostEqual(detections[0][1], 0.9, places=5) + self.assertAlmostEqual(detections[0][2], (center - box_h / 2) / 640, places=5) + self.assertAlmostEqual(detections[0][3], (center - box_w / 2) / 640, places=5) + self.assertAlmostEqual(detections[0][4], (center + box_h / 2) / 640, places=5) + self.assertAlmostEqual(detections[0][5], (center + box_w / 2) / 640, places=5) + + # remaining slots stay empty + self.assertTrue(np.all(detections[1:] == 0)) + + def test_uses_the_anchor_for_the_records_layer(self): + # the same box on stride 32 (layer 2) must decode to a larger box + def decode(layer_idx: int) -> np.ndarray: + outputs = [ + build_ppu_record( + box=(0.5, 0.5, 0.5, 0.5), + grid_y=1, + grid_x=1, + anchor_idx=0, + layer_idx=layer_idx, + score=0.9, + label=0, + ).reshape(1, 1, PPU_RECORD_SIZE) + ] + return decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + + small = decode(0) + large = decode(2) + + small_width = small[0][5] - small[0][3] + large_width = large[0][5] - large[0][3] + + self.assertGreater(large_width, small_width) + + def test_drops_detections_below_the_score_threshold(self): + outputs = [ + build_ppu_record( + box=(0.5, 0.5, 0.5, 0.5), + grid_y=10, + grid_x=10, + anchor_idx=0, + layer_idx=0, + score=0.1, + label=2, + ).reshape(1, 1, PPU_RECORD_SIZE) + ] + + detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_caps_output_at_twenty_detections(self): + records = np.stack( + [ + build_ppu_record( + box=(0.5, 0.5, 0.5, 0.5), + grid_y=i, + grid_x=i, + anchor_idx=0, + layer_idx=0, + score=0.9, + label=1, + ) + for i in range(40) + ] + ) + + detections = decode_ppu_anchor( + [records.reshape(1, 40, PPU_RECORD_SIZE)], 640, 640, 0.25, 0.45 + ) + + self.assertEqual(detections.shape, (20, 6)) + + def test_returns_empty_detections_for_no_output(self): + self.assertTrue(np.all(decode_ppu_anchor([], 640, 640, 0.25, 0.45) == 0)) + + def test_returns_empty_detections_for_an_unexpected_record_width(self): + # a non-PPU model would produce a different record width + outputs = [np.zeros((1, 4, 16), dtype=np.uint8)] + + self.assertTrue(np.all(decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) == 0)) + + +class TestDeepxModelFormats(unittest.TestCase): + def test_every_yolo_format_belongs_to_exactly_one_family(self): + families = [ANCHOR_FORMATS, ANCHOR_FREE_FORMATS, NMS_IN_HEAD_FORMATS] + yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS + + for fmt in ModelFormatEnum: + if fmt not in yolo_formats: + continue + + matches = [family for family in families if fmt in family] + self.assertEqual( + len(matches), 1, f"{fmt.value} must be in exactly one family" + ) + + def test_families_do_not_overlap(self): + self.assertFalse(ANCHOR_FORMATS & ANCHOR_FREE_FORMATS) + self.assertFalse(ANCHOR_FORMATS & NMS_IN_HEAD_FORMATS) + self.assertFalse(ANCHOR_FREE_FORMATS & NMS_IN_HEAD_FORMATS) + + def test_every_format_belongs_to_a_yolo_family(self): + # apart from the `auto` sentinel, every ModelFormatEnum member is a + # yolo-generic head shape -- nothing should be unclassified + yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS + + for fmt in ModelFormatEnum: + if fmt is ModelFormatEnum.auto: + continue + + self.assertIn(fmt, yolo_formats, f"{fmt.value} is not classified") + + def test_auto_is_in_no_family(self): + """`auto` means "no explicit layout", so every decode branch keyed off + a family must fall through to shape-based inference.""" + self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FORMATS) + self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FREE_FORMATS) + self.assertNotIn(ModelFormatEnum.auto, NMS_IN_HEAD_FORMATS) + + +class TestDeepxDetectorConfig(unittest.TestCase): + def test_ppu_without_model_format_is_rejected(self): + """PPU records are fixed-width regardless of variant, so an unset + model_format can't be inferred and must not silently fall back to the + anchor decoder.""" + with self.assertRaises(ValidationError): + DeepxDetectorConfig(type="deepx", ppu=True) + + def test_ppu_with_anchor_free_model_format_is_accepted(self): + config = DeepxDetectorConfig( + type="deepx", ppu=True, model_format=ModelFormatEnum.anchor_free + ) + + self.assertEqual(config.model_format, ModelFormatEnum.anchor_free) + + def test_raw_without_model_format_is_still_accepted(self): + # the raw path falls back to shape-based inference, so this remains + # optional when ppu is not set + config = DeepxDetectorConfig(type="deepx") + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_an_omitted_model_format_folds_to_auto(self): + """The config form round-trips a value for every field, so a blank or + null entry has to land on the sentinel rather than stay None.""" + for value in (None, "", " "): + with self.subTest(value=value): + config = DeepxDetectorConfig(type="deepx", model_format=value) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_ppu_with_damoyolo_model_type_is_rejected(self): + """No confirmed PPU record layout exists for DAMO-YOLO yet.""" + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + ppu=True, + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.damoyolo, + ) + + def test_ssd_model_type_takes_no_model_format(self): + """SSD's layout is fixed: DX-COM keeps the softmax and prior decode + inside the .dxnn, so no backbone has to be named.""" + config = DeepxDetectorConfig(type="deepx", model_type=DeepxModelTypeEnum.ssd) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + def test_ssd_model_type_rejects_a_model_format(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.ssd, + ) + + def test_damoyolo_model_type_rejects_a_model_format(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig( + type="deepx", + model_format=ModelFormatEnum.anchor_free, + model_type=DeepxModelTypeEnum.damoyolo, + ) + + def test_damoyolo_model_type_without_model_format_is_accepted(self): + config = DeepxDetectorConfig( + type="deepx", + model_type=DeepxModelTypeEnum.damoyolo, + ) + + self.assertIs(config.model_format, ModelFormatEnum.auto) + + +class TestDeepxDeviceIds(unittest.TestCase): + def test_a_comma_separated_string_is_kept(self): + config = DeepxDetectorConfig(type="deepx", device_ids="0,1") + + self.assertEqual(config.device_ids, "0,1") + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_a_yaml_list_is_accepted(self): + """A bare list is the natural thing to hand-write, so it is folded + into the same comma-separated form the config UI writes.""" + config = DeepxDetectorConfig(type="deepx", device_ids=[0, 1]) + + self.assertEqual(config.device_ids, "0,1") + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_a_single_index_is_accepted(self): + config = DeepxDetectorConfig(type="deepx", device_ids=1) + + self.assertEqual(config.device_ids, "1") + self.assertEqual(resolve_devices(config.device_ids), [1]) + + def test_surrounding_whitespace_is_tolerated(self): + config = DeepxDetectorConfig(type="deepx", device_ids=" 0 , 1 ") + + self.assertEqual(resolve_devices(config.device_ids), [0, 1]) + + def test_an_unparsable_value_is_rejected_at_config_load(self): + with self.assertRaises(ValidationError): + DeepxDetectorConfig(type="deepx", device_ids="gpu0") + + def test_unset_leaves_device_selection_to_auto_detection(self): + config = DeepxDetectorConfig(type="deepx") + + self.assertEqual(config.device_ids, DEVICE_IDS_AUTO) + self.assertEqual(parse_device_ids(config.device_ids), []) + + def test_a_blank_entry_folds_to_auto(self): + """The config form round-trips a value for every field, so a blank or + null entry has to land on the sentinel rather than stay None.""" + for value in (None, "", " ", []): + with self.subTest(value=value): + config = DeepxDetectorConfig(type="deepx", device_ids=value) + + self.assertEqual(config.device_ids, DEVICE_IDS_AUTO) + + def test_auto_is_case_insensitive(self): + config = DeepxDetectorConfig(type="deepx", device_ids="AUTO") + + self.assertEqual(parse_device_ids(config.device_ids), []) + + +class TestDeepxPpuAnchorFreeDecode(unittest.TestCase): + def test_reads_the_box_bytes_as_pixel_geometry(self): + """An anchor-free PPU record holds cx, cy, w, h directly in pixels.""" + record = build_ppu_record( + box=(320.0, 160.0, 64.0, 32.0), + # grid columns carry no meaning for an anchor-free head, and must + # not influence the result + grid_y=7, + grid_x=9, + anchor_idx=2, + layer_idx=2, + score=0.9, + label=0, + ) + outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)] + + detections = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) + + # x: 320 +/- 32 -> 288..352, y: 160 +/- 16 -> 144..176 + self.assertAlmostEqual(detections[0][2], 144 / 640, places=5) + self.assertAlmostEqual(detections[0][3], 288 / 640, places=5) + self.assertAlmostEqual(detections[0][4], 176 / 640, places=5) + self.assertAlmostEqual(detections[0][5], 352 / 640, places=5) + + def test_anchor_formula_would_blow_the_box_up(self): + """Guards the bug this decoder exists to prevent.""" + record = build_ppu_record( + box=(320.0, 160.0, 64.0, 32.0), + grid_y=7, + grid_x=9, + anchor_idx=2, + layer_idx=2, + score=0.9, + label=0, + ) + outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)] + + anchor_free = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45) + anchor = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) + + # the anchor decode squares the width, saturating the clip to the frame + self.assertEqual(anchor[0][5], 1.0) + self.assertLess(anchor_free[0][5], 1.0) + + +class TestDeepxRawAnchorDecode(unittest.TestCase): + def build_raw_anchor_output(self, rows): + """Build an (1, N, 5+C) tensor with 80 classes.""" + out = np.zeros((1, len(rows), 85), dtype=np.float32) + + for i, (cx, cy, w, h, obj, label, cls_score) in enumerate(rows): + out[0, i, 0:4] = [cx, cy, w, h] + out[0, i, 4] = obj + out[0, i, 5 + label] = cls_score + + return [out] + + def test_confidence_is_objectness_times_class_score(self): + outputs = self.build_raw_anchor_output( + [(320.0, 320.0, 40.0, 80.0, 0.8, 3, 0.5)] + ) + + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) + + self.assertEqual(detections[0][0], 3) + self.assertAlmostEqual(detections[0][1], 0.4, places=5) + + def test_drops_rows_whose_combined_score_is_below_threshold(self): + # 0.4 * 0.5 = 0.2, under a 0.25 threshold even though both parts are + # individually above it + outputs = self.build_raw_anchor_output( + [(320.0, 320.0, 40.0, 80.0, 0.4, 3, 0.5)] + ) + + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_converts_center_form_to_normalized_corners(self): + outputs = self.build_raw_anchor_output( + [(320.0, 160.0, 64.0, 32.0, 1.0, 0, 1.0)] + ) + + detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45) + + self.assertAlmostEqual(detections[0][2], 144 / 640, places=5) + self.assertAlmostEqual(detections[0][3], 288 / 640, places=5) + self.assertAlmostEqual(detections[0][4], 176 / 640, places=5) + self.assertAlmostEqual(detections[0][5], 352 / 640, places=5) + + +class TestDeepxRawAnchorFreeDecode(unittest.TestCase): + def build_raw_anchor_free_output(self, rows, num_classes=3, total_rows=10): + """Build an (1, N, 4+C) tensor -- already row-per-box -- with no + objectness column, padded with all-zero rows so N comfortably + outnumbers the channel count (4+C). + + The decoder tells a channel-major export apart from a row-major one + purely by comparing those two sizes, the same way it would on a real + grid of thousands of anchors -- and that comparison only comes out + right when boxes outnumber channels, so a test with too few rows + would trip the same ambiguity `auto` accepts by design. + """ + total_rows = max(total_rows, len(rows)) + out = np.zeros((1, total_rows, 4 + num_classes), dtype=np.float32) + + for i, (cx, cy, w, h, label, cls_score) in enumerate(rows): + out[0, i, 0:4] = [cx, cy, w, h] + out[0, i, 4 + label] = cls_score + + return [out] + + def test_keeps_a_detection_between_the_deepx_and_frigate_default_thresholds(self): + """Regression test: post_process_yolo hardcodes a 0.4 score + threshold, which would silently drop this 0.3-confidence detection + even though it clears the DEEPX decoder's own 0.25 default.""" + outputs = self.build_raw_anchor_free_output( + [(320.0, 320.0, 40.0, 80.0, 2, 0.3)] + ) + + detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45) + + self.assertEqual(detections[0][0], 2) + self.assertAlmostEqual(detections[0][1], 0.3, places=5) + + def test_drops_detections_below_the_score_threshold(self): + outputs = self.build_raw_anchor_free_output( + [(320.0, 320.0, 40.0, 80.0, 2, 0.1)] + ) + + detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_transposes_a_channel_major_output(self): + """DX-COM may export (4+C, N) instead of (N, 4+C); both must decode + to the same result.""" + row_major = self.build_raw_anchor_free_output( + [(320.0, 320.0, 40.0, 80.0, 2, 0.9)] + ) + channel_major = [np.transpose(row_major[0], (0, 2, 1))] + + row_major_detections = decode_raw_anchor_free(row_major, 640, 640, 0.25, 0.45) + channel_major_detections = decode_raw_anchor_free( + channel_major, 640, 640, 0.25, 0.45 + ) + + np.testing.assert_array_almost_equal( + row_major_detections, channel_major_detections + ) + + def test_converts_center_form_to_normalized_corners(self): + outputs = self.build_raw_anchor_free_output( + [(320.0, 160.0, 64.0, 32.0, 0, 1.0)] + ) + + detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45) + + self.assertAlmostEqual(detections[0][2], 144 / 640, places=5) + self.assertAlmostEqual(detections[0][3], 288 / 640, places=5) + self.assertAlmostEqual(detections[0][4], 176 / 640, places=5) + self.assertAlmostEqual(detections[0][5], 352 / 640, places=5) + + def test_returns_empty_detections_for_no_output(self): + out = np.zeros((1, 0, 84), dtype=np.float32) + + self.assertTrue(np.all(decode_raw_anchor_free([out], 640, 640, 0.25, 0.45) == 0)) + + +class TestDeepxRawNmsInHeadDecode(unittest.TestCase): + def test_reads_corner_records_without_running_nms(self): + """Two heavily overlapping boxes both survive: the head already ran NMS.""" + out = np.array( + [ + [ + [100.0, 100.0, 200.0, 200.0, 0.9, 2.0], + [102.0, 102.0, 202.0, 202.0, 0.8, 2.0], + ] + ], + dtype=np.float32, + ) + + detections = decode_raw_nms_in_head([out], 640, 640, 0.25) + + self.assertEqual(detections[0][0], 2) + self.assertAlmostEqual(detections[0][1], 0.9, places=5) + self.assertAlmostEqual(detections[0][3], 100 / 640, places=5) + self.assertEqual(detections[1][0], 2) + self.assertAlmostEqual(detections[1][1], 0.8, places=5) + + def test_applies_the_score_threshold(self): + out = np.array( + [ + [ + [100.0, 100.0, 200.0, 200.0, 0.9, 2.0], + [300.0, 300.0, 400.0, 400.0, 0.1, 5.0], + ] + ], + dtype=np.float32, + ) + + detections = decode_raw_nms_in_head([out], 640, 640, 0.25) + + self.assertAlmostEqual(detections[0][1], 0.9, places=5) + self.assertTrue(np.all(detections[1] == 0)) + + def test_returns_empty_detections_for_an_empty_output(self): + out = np.zeros((1, 0, 6), dtype=np.float32) + + self.assertTrue(np.all(decode_raw_nms_in_head([out], 640, 640, 0.25) == 0)) + + +class TestDeepxSsdDecode(unittest.TestCase): + """The .dxnn emits final boxes and probabilities, so these feed the + decoder what a real compiled SSD produces: corner-form boxes normalized + to [0, 1] and post-softmax confidences with background at column 0.""" + + def build_ssd_output(self, box_index, box, class_id, prob, num_boxes=3000): + num_classes = 21 # 20 VOC classes + background + + boxes = np.zeros((1, num_boxes, 4), dtype=np.float32) + boxes[0, box_index] = box + + # every other box is background, as the model would report it + scores = np.zeros((1, num_boxes, num_classes), dtype=np.float32) + scores[0, :, 0] = 1.0 + scores[0, box_index, 0] = 1.0 - prob + scores[0, box_index, class_id] = prob + + return [boxes, scores] + + def test_decodes_an_already_decoded_box(self): + outputs = self.build_ssd_output( + box_index=7, box=(0.1, 0.2, 0.5, 0.8), class_id=5, prob=0.9 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + # boxes pass through untouched apart from the corner reorder, and + # the background class is dropped so labels shift down by one + self.assertEqual(detections[0][0], 4) + self.assertAlmostEqual(float(detections[0][1]), 0.9, places=5) + self.assertAlmostEqual(float(detections[0][2]), 0.2, places=5) # y_min + self.assertAlmostEqual(float(detections[0][3]), 0.1, places=5) # x_min + self.assertAlmostEqual(float(detections[0][4]), 0.8, places=5) # y_max + self.assertAlmostEqual(float(detections[0][5]), 0.5, places=5) # x_max + + def test_confidences_are_not_softmaxed_again(self): + """A second softmax over 21 already-normalized probabilities pulls + every score toward 1/21, which is what silently dropped every real + detection under the default threshold.""" + outputs = self.build_ssd_output( + box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=15, prob=0.6 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertAlmostEqual(float(detections[0][1]), 0.6, places=5) + + def test_any_box_count_is_accepted(self): + """3000 for the MobileNet backbones, 8732 for VGG16 -- the decoder + reads whatever the model emits rather than a per-backbone table.""" + for num_boxes in (3000, 8732): + with self.subTest(num_boxes=num_boxes): + outputs = self.build_ssd_output( + box_index=num_boxes - 1, + box=(0.0, 0.0, 1.0, 1.0), + class_id=1, + prob=0.8, + num_boxes=num_boxes, + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertEqual(detections[0][0], 0) + self.assertAlmostEqual(float(detections[0][1]), 0.8, places=5) + + def test_output_order_does_not_matter(self): + boxes, scores = self.build_ssd_output( + box_index=1, box=(0.2, 0.2, 0.6, 0.6), class_id=3, prob=0.7 + ) + + forward = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45) + reversed_ = decode_ssd_raw([scores, boxes], 300, 300, 0.25, 0.45) + + np.testing.assert_array_equal(forward, reversed_) + + def test_low_confidence_detection_is_dropped(self): + outputs = self.build_ssd_output( + box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=3, prob=0.05 + ) + + detections = decode_ssd_raw(outputs, 300, 300, 0.5, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_outputs_cannot_be_told_apart(self): + # two tensors that both look class-shaped (last dim != 4) + outputs = [ + np.zeros((1, 10, 21), dtype=np.float32), + np.zeros((1, 10, 21), dtype=np.float32), + ] + + detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_box_and_class_counts_disagree(self): + boxes = np.zeros((1, 10, 4), dtype=np.float32) + scores = np.zeros((1, 12, 21), dtype=np.float32) + + detections = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + +class TestDeepxDamoyoloDecode(unittest.TestCase): + def total_priors(self, width, height, strides): + return sum((height // s) * (width // s) for s in strides) + + def test_decodes_a_detection_from_already_decoded_boxes(self): + """box_output shaped (1, N, 4): DX-COM's export is assumed to have + already applied the full distance-to-box decode, so these are final + (x_min, y_min, x_max, y_max) pixel coordinates, used as-is.""" + width = height = 640 + num_priors = 100 + prior_index = 5 + + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, prior_index, 7] = 0.9 + + box_output = np.zeros((1, num_priors, 4), dtype=np.float32) + box_output[0, prior_index] = [100.0, 120.0, 140.0, 160.0] + + detections = decode_damoyolo_raw( + [cls_scores, box_output], width, height, 0.25, 0.45 + ) + + self.assertEqual(detections[0][0], 7) + self.assertAlmostEqual(detections[0][1], 0.9, places=5) + self.assertAlmostEqual(detections[0][2], 120 / height, places=5) + self.assertAlmostEqual(detections[0][3], 100 / width, places=5) + self.assertAlmostEqual(detections[0][4], 160 / height, places=5) + self.assertAlmostEqual(detections[0][5], 140 / width, places=5) + + def test_decodes_a_detection_from_a_dfl_distribution(self): + """box_output shaped (1, N, 4, bins): a raw per-side DFL histogram, + needing softmax + integral + stride-scaled distance-to-box here.""" + width = height = 640 + strides = (8, 16, 32) + grid_w = width // 8 + prior_row, prior_col = 10, 10 + prior_index = prior_row * grid_w + prior_col + num_priors = self.total_priors(width, height, strides) + + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, prior_index, 9] = 0.8 + + # reg_max=1 (2 bins): put ~all softmax mass on bin index 1, so the + # integral (weighted sum against [0, 1]) comes out to ~1.0 stride + # unit on every side + box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32) + box_output[0, prior_index, :, 0] = -30.0 + box_output[0, prior_index, :, 1] = 30.0 + + detections = decode_damoyolo_raw( + [cls_scores, box_output], width, height, 0.25, 0.45, strides + ) + + # stride-8 grid cell (10, 10) centers at pixel (80, 80); ~1 + # stride-unit distance on every side -> an 8px box around it + self.assertEqual(detections[0][0], 9) + self.assertAlmostEqual(detections[0][1], 0.8, places=5) + self.assertAlmostEqual(detections[0][2], 72 / height, places=3) + self.assertAlmostEqual(detections[0][3], 72 / width, places=3) + self.assertAlmostEqual(detections[0][4], 88 / height, places=3) + self.assertAlmostEqual(detections[0][5], 88 / width, places=3) + + def test_returns_empty_detections_below_score_threshold(self): + num_priors = 10 + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.1 + + box_output = np.zeros((1, num_priors, 4), dtype=np.float32) + box_output[0, 0] = [10.0, 10.0, 20.0, 20.0] + + detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.5, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_dfl_prior_count_does_not_match_the_grid( + self, + ): + # a DFL-shaped box output with far fewer priors than the (8, 16, 32) + # strides actually produce for a 640x640 input + num_priors = 5 + cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.9 + box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32) + + detections = decode_damoyolo_raw( + [cls_scores, box_output], 640, 640, 0.25, 0.45, (8, 16, 32) + ) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_when_box_and_class_counts_disagree(self): + cls_scores = np.zeros((1, 10, 80), dtype=np.float32) + cls_scores[0, 0, 1] = 0.9 + box_output = np.zeros((1, 7, 4), dtype=np.float32) + + detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.25, 0.45) + + self.assertTrue(np.all(detections == 0)) + + def test_returns_empty_detections_for_wrong_output_count(self): + outputs = [np.zeros((1, 10, 80), dtype=np.float32)] + + detections = decode_damoyolo_raw(outputs, 640, 640, 0.25, 0.45, (8, 16, 32)) + + self.assertTrue(np.all(detections == 0)) + + +if __name__ == "__main__": + unittest.main() + + +class TestDeepxModelType(unittest.TestCase): + def test_model_type_defaults_to_yolo_generic(self): + """Frigate's own model_type default is ssd, which silently picked the + SSD decoder for anyone who left it unset.""" + config = DeepxDetectorConfig(type="deepx") + + self.assertIs(config.model_type, DeepxModelTypeEnum.yologeneric) + + def test_model_type_values_match_frigate_model_types(self): + """The detector writes its choice back onto model.model_type, so every + value has to round-trip through ModelTypeEnum.""" + for value in DeepxModelTypeEnum: + with self.subTest(value=value): + self.assertEqual(ModelTypeEnum(value.value).value, value.value) diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index 4eafe4537b..ecb162af2d 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -322,7 +322,7 @@ }, "model_type": { "label": "Object Detection Model Type", - "description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization." + "description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization." } }, "model_path": { @@ -357,6 +357,32 @@ "description": "Optional API key for authenticated DeepStack services." } }, + "deepx": { + "label": "DEEPX NPU", + "description": "DEEPX NPU detector running .dxnn models via the DX-RT runtime.", + "device_ids": { + "label": "DEEPX device indices", + "description": "Comma-separated NPU device indices this detector should bind to, e.g. \"0,1\". Leave as \"auto\" to detect them from the DXNN_DEVICES environment variable, then the DX-RT device count, then a scan of /dev/dxrt*." + }, + "model_type": { + "label": "Detection model architecture", + "description": "Which decoder reads the model's output. Overrides model.model_type for this detector, whose default of ssd would otherwise apply whenever the root model block leaves it unset." + }, + "ppu": { + "label": "PPU-compiled model", + "description": "Set to true when the model was compiled with DEEPX's Post-Processing Unit support, which moves candidate selection onto the NPU and changes the output format. Only supported for model_type yolo-generic." + }, + "model_format": { + "label": "Detection head layout the model was compiled from", + "description": "Selects how the model's output is decoded. Only used for model_type yolo-generic: leave as \"auto\" to infer the layout from the output shape, which cannot tell an anchor-based head from an anchor-free one and is only reliable for anchor-free heads. Required when ppu is true, since a PPU record's layout is identical for every yolo-generic variant and cannot be inferred. ssd and damo-yolo each read one fixed layout and take no value here." + }, + "score_threshold": { + "label": "Score threshold used when decoding detector output" + }, + "nms_threshold": { + "label": "NMS IoU threshold used when decoding detector output" + } + }, "edgetpu": { "label": "EdgeTPU", "description": "EdgeTPU detector that runs TensorFlow Lite models compiled for Coral EdgeTPU using the EdgeTPU delegate.", @@ -479,7 +505,7 @@ }, "model_type": { "label": "Object Detection Model Type", - "description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization." + "description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization." } }, "genai": {