nik/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-10-18 07:29:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge eccf36f15eefcdbb36b07b77adb83a1b159bfe8a into d37dff1b491e752d7bf6b0ca2c7efbaedca623cd

Browse Source
This commit is contained in:
A. Ahmet 2026-08-29 23:06:27 +00:00 committed by GitHub
commit 6a4995a1e7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
27 changed files with 2598 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CODEOWNERS
View File

@ -5,3 +5,4 @@
/docker/rockchip/ @MarcA711
/docker/rocm/ @harakas
/docker/hailo8l/ @spanner3003
/docker/deepx/ @sixfab

105
docker/deepx/user_installation.sh Executable file
View File

@ -0,0 +1,105 @@
#!/bin/bash
# Installs the DEEPX NPU kernel driver on the Docker host.
#
# The driver has to be built and loaded on the host rather than in the Frigate
# container: containers share the host kernel, so a kernel module cannot be
# loaded from inside one. Once loaded it exposes the NPU as /dev/dxrt* device
# nodes, which are passed through to the container (see the DEEPX section of
# the installation docs).
#
# A DEEPX install has three separately versioned pieces that have to agree:
#
# DX-RT runtime v3.4.2 the Frigate image, docker/main/install_dxrt.sh
# Kernel driver v2.6.0 this file (provides dxrt_driver and dx_dma)
# NPU firmware v2.7.4 the module itself, flashed from the host
#
# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the
# versions pinned here sit above that floor. A mismatch is the most common
# cause of inference requests that are accepted but never complete, and it
# fails silently rather than at startup. This script only installs the driver:
# reading or flashing the firmware version requires the DX-RT runtime on the
# host, which conflicts with the copy Frigate runs in its container, so that is
# a separate manual step covered in the installation docs.
#
# DEEPX NPU support in Frigate is developed and maintained by Sixfab
# (https://sixfab.com).
set -euo pipefail
driver_version="v2.6.0"
# Commit the tag above resolves to, pinned so a later retag or repoint of
# v2.6.0 upstream can't silently swap in different source between review and
# build, before it's compiled and installed as root. DEEPX doesn't sign tags
# or publish checksums for this repo, so an immutable commit is the strongest
# verification available. Update both together when bumping driver_version:
# git ls-remote https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git "refs/tags/<version>"
driver_commit="7074748e7104f470b02f517583abba652b3f05fa"
firmware_version="v2.7.4"
sudo apt-get update
sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils
if ! lspci -d 1ff4: | grep -q .; then
echo "No DEEPX device found on the PCIe bus (lspci -d 1ff4:)."
echo "Check that the module is seated correctly before continuing."
exit 1
fi
# Fetch the pinned commit directly (GitHub allows fetching a public repo by
# SHA, not just by branch/tag) rather than cloning "${driver_version}", so
# driver_commit above is what actually determines the source, not a ref that
# could move after this script was reviewed.
mkdir dx_rt_npu_linux_driver
cd dx_rt_npu_linux_driver
git init -q
git remote add origin https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git
git fetch --depth 1 origin "${driver_commit}"
git checkout -q FETCH_HEAD
fetched_commit=$(git rev-parse HEAD)
if [[ "${fetched_commit}" != "${driver_commit}" ]]; then
echo "Fetched commit ${fetched_commit} does not match pinned driver_commit ${driver_commit}."
echo "Refusing to build unverified driver source."
exit 1
fi
cd modules
# --reload unloads any running modules and loads the newly built ones, so the
# NPU is usable without a reboot.
sudo ./build.sh -c install --reload
sudo depmod -A
# dx_dma is the PCIe transport, dxrt_driver is the NPU driver on top of it.
for module in dx_dma dxrt_driver; do
if ! sudo modprobe "${module}"; then
echo "Unable to load the ${module} kernel module, common reasons are:"
echo "- Secure Boot is enabled and is rejecting the unsigned module."
echo "- The running kernel does not match the installed linux-headers."
exit 1
fi
done
if ! compgen -G "/dev/dxrt*" > /dev/null; then
echo "Modules loaded but no /dev/dxrt* device node appeared."
echo "Run ./sanity_check.sh from the driver repo to diagnose."
exit 1
fi
# dxrtd runs inside the Frigate container and may only run once per system, so
# a host copy left over from a runtime install has to be disabled.
if systemctl list-unit-files dxrt.service &> /dev/null; then
echo "Disabling the host dxrt.service so the container can run dxrtd."
sudo systemctl disable --now dxrt.service
fi
echo "DEEPX driver installation complete."
echo "Driver version: $(modinfo -F version dxrt_driver)"
echo "Device node(s): $(echo /dev/dxrt*)"
echo
echo "This driver expects NPU firmware ${firmware_version}. The firmware version"
echo "can only be read with the DX-RT runtime installed on the host, so check it"
echo "before starting Frigate, then remove the host runtime: Frigate runs its own"
echo "copy of dxrtd and only one may run per system."

4
docker/main/Dockerfile
View File

@ -285,6 +285,10 @@ ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/lib/axcl"
RUN --mount=type=bind,source=docker/main/install_memryx.sh,target=/deps/install_memryx.sh \
bash -c "bash /deps/install_memryx.sh"
# Install DEEPX DX-RT runtime (dxrtd daemon + dx_engine python bindings)
RUN --mount=type=bind,source=docker/main/install_dxrt.sh,target=/deps/install_dxrt.sh \
/deps/install_dxrt.sh
COPY --from=deps-rootfs / /
RUN ldconfig

87
docker/main/install_dxrt.sh Executable file
View File

@ -0,0 +1,87 @@
#!/bin/bash
set -euxo pipefail
# DEEPX DX-RT runtime (dxrtd daemon + the dx_engine python bindings) used by
# the `deepx` detector plugin.
#
# A DEEPX install has three separately versioned pieces that have to agree:
#
# DX-RT runtime v3.4.2 this file, installed into the Frigate image
# Kernel driver v2.6.0 the host, docker/deepx/user_installation.sh
# NPU firmware v2.7.4 the module itself, flashed from the host
#
# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the
# versions above sit just over that floor. The runtime is pinned rather than
# resolved to "latest" so image builds stay reproducible, and so there is a
# known version for the host side to match. A mismatch usually presents as
# inference requests that are accepted but never complete, not as a startup
# error, so bump all three together and keep the table in
# docs/docs/frigate/installation.md in sync.
dxrt_version="v3.4.2"
# SHA-256 of the .deb pinned above, one per architecture Frigate builds for.
# DEEPX doesn't publish checksums or sign this release, so these are computed
# once from a vetted download and hardcoded here as the actual integrity
# check: a download that doesn't match one of these hashes is rejected before
# it reaches apt-get, regardless of whether the mismatch came from the v3.4.2
# tag being retargeted or the file being replaced upstream. Recompute both
# (`sha256sum libdxrt-bin_*.deb`) when bumping dxrt_version.
declare -A dxrt_sha256=(
[amd64]="da782deac10acb1c997b52ea6d0160c06f112a68ad5e25035cfc8e16de0a8bdc"
[arm64]="161223058938c54efadc581b33499210194da58e55cd5ade332e692ca0d649c2"
)
# The release directory and the package filename both use the bare version,
# while the git tag carries a "v".
dxrt_release="${dxrt_version#v}"
deb_arch=$(dpkg --print-architecture)
deb_file="/tmp/libdxrt-bin_${dxrt_release}_${deb_arch}.deb"
# Fetch just the one package rather than the source archive: the repository
# keeps every past release under release/, so the tag archive is several
# hundred MB of .deb files we would immediately throw away. Note that
# release/latest is a symlink and would not survive the archive anyway.
wget -qO "${deb_file}" \
"https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${dxrt_version}/release/${dxrt_release}/libdxrt-bin_${dxrt_release}_${deb_arch}.deb"
expected_sha256="${dxrt_sha256[${deb_arch}]:-}"
if [[ -z "${expected_sha256}" ]]; then
echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install an unverified package."
exit 1
fi
actual_sha256=$(sha256sum "${deb_file}" | cut -d' ' -f1)
if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then
echo "SHA-256 mismatch for ${deb_file}:"
echo " expected ${expected_sha256}"
echo " actual ${actual_sha256}"
echo "Refusing to install a package that doesn't match the pinned checksum."
exit 1
fi
# Install with dpkg rather than apt. install_deps.sh finishes by purging gnupg
# and clearing /var/lib/apt/lists, so apt here would have to re-fetch about
# 10 MB of package indexes over the network before it could resolve anything,
# which is a build failure waiting to happen for no benefit. The package only
# depends on libc6, libgcc-s1, libncurses6, libstdc++6 and libtinfo6, all of
# which the deps stage already has, and its postinst does nothing but run
# ldconfig and print where the files went. dpkg fails loudly if a dependency
# ever goes missing.
dpkg -i "${deb_file}"
# The package bundles a dx_engine wheel per cpython ABI tag. Ask dpkg what it
# installed rather than hardcoding a path, so this keeps working if DEEPX moves
# the wheels between releases.
package=$(dpkg-deb -f "${deb_file}" Package)
py_tag="cp$(python3 -c 'import sys; print(f"{sys.version_info[0]}{sys.version_info[1]}")')"
wheel=$(dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" | grep -- "${py_tag}" | sort | head -1 || true)
if [[ -z "${wheel}" ]]; then
echo "No dx_engine wheel for ${py_tag} in ${package}. Wheels available:"
dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" || echo " (none)"
exit 1
fi
pip3 install "${wheel}"
rm -f "${deb_file}"

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/consumer-for Normal file
View File

@ -0,0 +1 @@
dxrtd

0
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/dependencies.d/log-prepare Normal file
View File

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/pipeline-name Normal file
View File

@ -0,0 +1 @@
dxrtd-pipeline

4
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/run Executable file
View File

@ -0,0 +1,4 @@
#!/command/with-contenv bash
# shellcheck shell=bash
exec logutil-service /dev/shm/logs/dxrtd

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd-log/type Normal file
View File

@ -0,0 +1 @@
longrun

0
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/dependencies.d/prepare Normal file
View File

43
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/finish Executable file
View File

@ -0,0 +1,43 @@
#!/command/with-contenv bash
# shellcheck shell=bash
# Take down the S6 supervision tree when the service fails
set -o errexit -o nounset -o pipefail
# Logs should be sent to stdout so that s6 can collect them
declare exit_code_container
exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
readonly exit_code_container
readonly exit_code_service="${1}"
readonly exit_code_signal="${2}"
readonly service="dxrtd"
echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
# dxrtd takes itself down when the machine has no DEEPX NPU, which is the
# normal case: this service ships in the default image and most users do not
# have the hardware. That self-shutdown reaches this script as exit 256 /
# signal 15, indistinguishable from an operator stopping the container, so
# without the marker written by run every machine lacking a DEEPX NPU would
# have Frigate halted at startup.
if [[ -f /dev/shm/.dxrtd-no-hardware ]]; then
echo "[INFO] Service ${service} stopped because no DEEPX NPU is present; leaving the rest of Frigate running"
exit 0
fi
# Past this point the service really was running, so only a genuine failure
# should bring the container down.
if [[ "${exit_code_service}" -eq 256 ]]; then
if [[ "${exit_code_container}" -eq 0 ]]; then
echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
fi
if [[ "${exit_code_signal}" -eq 15 ]]; then
exec /run/s6/basedir/bin/halt
fi
elif [[ "${exit_code_service}" -ne 0 ]]; then
if [[ "${exit_code_container}" -eq 0 ]]; then
echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
fi
exec /run/s6/basedir/bin/halt
fi

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/producer-for Normal file
View File

@ -0,0 +1 @@
dxrtd-log

41
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/run Executable file
View File

@ -0,0 +1,41 @@
#!/command/with-contenv bash
# shellcheck shell=bash
# Start the DEEPX DX-RT runtime daemon (dxrtd)
set -o errexit -o nounset -o pipefail
# Logs should be sent to stdout so that s6 can collect them
exec 2>&1
# The DEEPX kernel driver runs on the Docker host, not in this container, and
# exposes the NPU as /dev/dxrt* device nodes that are passed through. Without
# them there is no NPU to serve, so exit cleanly and let the rest of Frigate
# start normally: this service ships in the default image and most users do
# not have DEEPX hardware.
if ! compgen -G "/dev/dxrt*" > /dev/null; then
echo "[INFO] No DEEPX NPU device found, not starting dxrtd"
# Tell finish that this shutdown is deliberate. Taking the service down
# here arrives at finish as exit 256 / signal 15, which is exactly what an
# operator stopping the container looks like, so the marker is the only
# thing that tells the two apart.
touch /dev/shm/.dxrtd-no-hardware
exec s6-svc -Od .
fi
# Reaching here means there is hardware to serve, so clear any marker left by
# an earlier run before the device appeared.
rm -f /dev/shm/.dxrtd-no-hardware
# dxrtd is a system-wide singleton. If the host's own dxrt.service is still
# running it will refuse to start, so surface that up front rather than
# leaving the user to decode the daemon's own error.
if pgrep -x dxrtd > /dev/null; then
echo "[ERROR] Another dxrtd instance is already running"
echo "[ERROR] dxrtd may only run once per system. Stop the host service with:"
echo "[ERROR] sudo systemctl disable --now dxrt.service"
exit 1
fi
echo "[INFO] Starting dxrtd for DEEPX NPU device(s): $(echo /dev/dxrt*)"
exec /usr/local/bin/dxrtd

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/timeout-kill Normal file
View File

@ -0,0 +1 @@
30000

1
docker/main/rootfs/etc/s6-overlay/s6-rc.d/dxrtd/type Normal file
View File

@ -0,0 +1 @@
longrun

0
docker/main/rootfs/etc/s6-overlay/s6-rc.d/frigate/dependencies.d/dxrtd Normal file
View File

2
docker/main/rootfs/etc/s6-overlay/s6-rc.d/log-prepare/run
View File

@ -4,7 +4,7 @@
set -o errexit -o nounset -o pipefail
dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync)
dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/dxrtd)
mkdir -p "${dirs[@]}"
chown nobody:nogroup "${dirs[@]}"

0
docker/main/rootfs/etc/s6-overlay/s6-rc.d/user/contents.d/dxrtd-pipeline Normal file
View File

20
docker/main/rootfs/labelmap/voc-20.txt Normal file
View File

@ -0,0 +1,20 @@
0 aeroplane
1 bicycle
2 bird
3 boat
4 bottle
5 bus
6 car
7 cat
8 chair
9 cow
10 diningtable
11 dog
12 horse
13 motorbike
14 person
15 pottedplant
16 sheep
17 sofa
18 train
19 tvmonitor

123
docs/data/object_detectors_models.yaml
View File

@ -1269,3 +1269,126 @@ axengine:
input_dtype: int
input_pixel_format: bgr
labelmap_path: /labelmap/coco-80.txt
deepx:
title: DEEPX NPU
models:
- key: yolo
label: YOLO (anchor / anchor-free / NMS-in-head)
recommended: true
download: No model is bundled with Frigate. Download a pre-compiled `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point `model.path` at it. Set `model_format` to match the detection-head layout the model was compiled from.
ui: |-
Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings:
| Field | Value |
| ---------------------------------------- | ------------------------- |
| **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` |
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
| **Object detection model input width** | `640` |
| **Object detection model input height** | `640` |
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
| **Model Input D Type** | `int` (Frigate's default value) |
The width and height must match the resolution the `.dxnn` file was compiled for. Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden.
Set `model_type` and `model_format` on the detector itself rather than in the model settings above. `model_type` is `yolo-generic` here; `model_format` is the detection-head layout the model was compiled from (`anchor`, `anchor_free`, or `nms_in_head`), and leaving it at its `auto` default falls back to inferring the layout from the output shape, which cannot tell an anchor-based head from an anchor-free one.
yaml: |-
detectors:
deepx:
type: deepx
# Which decoder reads the model's output.
model_type: yolo-generic
# The detection-head layout the .dxnn was compiled from. Selects
# how the output is decoded. Leave as "auto" to infer it from
# the output shape, which cannot tell an anchor-based head from
# an anchor-free one.
model_format: anchor_free
# Optional. Comma-separated NPU device indices to bind to.
# Defaults to "auto", which binds to every NPU found.
# device_ids: "0"
# Set to true when the model was compiled with PPU support, which
# moves candidate selection onto the NPU.
# ppu: false
model:
path: /config/model_cache/deepx/model.dxnn
width: 640
height: 640
labelmap_path: /labelmap/coco-80.txt
- key: ssd
label: SSD (MobileNetV1 / MobileNetV2-Lite / VGG-16)
recommended: false
download: No model is bundled with Frigate. Download a pre-compiled SSD `.dxnn` model (MobileNetV1, MobileNetV2-Lite, or VGG-16 backbone) from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point `model.path` at it. These models are trained on Pascal VOC's 20 classes, not COCO, so `labelmap_path` must be set to `/labelmap/voc-20.txt`.
ui: |-
Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings:
| Field | Value |
| ---------------------------------------- | ------------------------- |
| **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` |
| **Label map for custom object detector** | `/labelmap/voc-20.txt` |
| **Object detection model input width** | `300` |
| **Object detection model input height** | `300` |
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
| **Model Input D Type** | `int` (Frigate's default value) |
Set `model_type` to `ssd` on the detector itself rather than in the model settings above. No `model_format` is needed: DX-COM keeps SSD's softmax and prior-box decode inside the compiled model, so the same decoder reads all three backbones.
yaml: |-
detectors:
deepx:
type: deepx
# Which decoder reads the model's output. The .dxnn already
# applies the softmax and prior-box decode, so no model_format
# is needed for any of the three backbones.
model_type: ssd
# Optional. Comma-separated NPU device indices to bind to.
# Defaults to "auto", which binds to every NPU found.
# device_ids: "0"
model:
path: /config/model_cache/deepx/model.dxnn
width: 300
height: 300
labelmap_path: /labelmap/voc-20.txt
- key: damoyolo
label: DAMO-YOLO (TinyNAS-L20T / S / M / L)
recommended: false
download: No model is bundled with Frigate. Download a pre-compiled DAMO-YOLO `.dxnn` model (TinyNAS-L20T, S, M, or L size) from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point `model.path` at it. All four sizes share one detection head, so no `model_format` is needed.
ui: |-
Navigate to **Settings > System > Detectors and model** and select **DEEPX NPU** from the detector type dropdown and click **Add**. Then on the same page, in the **Custom Model** tab, configure the model settings:
| Field | Value |
| ---------------------------------------- | ------------------------- |
| **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` |
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
| **Object detection model input width** | `640` |
| **Object detection model input height** | `640` |
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
| **Model Input D Type** | `int` (Frigate's default value) |
Set `model_type` to `damo-yolo` on the detector itself rather than in the model settings above.
yaml: |-
detectors:
deepx:
type: deepx
# Which decoder reads the model's output.
model_type: damo-yolo
# Optional. Comma-separated NPU device indices to bind to.
# Defaults to "auto", which binds to every NPU found.
# device_ids: "0"
model:
path: /config/model_cache/deepx/model.dxnn
width: 640
height: 640
labelmap_path: /labelmap/coco-80.txt

127
docs/docs/configuration/object_detectors.md
View File

@ -24,6 +24,7 @@ Frigate supports multiple different detectors that work on different types of ha
- [Coral EdgeTPU](#edge-tpu-detector): The Google Coral EdgeTPU is available in USB, Mini PCIe, and m.2 formats allowing for a wide range of compatibility with devices.
- [Hailo](#hailo-8): The Hailo8 and Hailo8L AI Acceleration module is available in m.2 format with a HAT for RPi devices, offering a wide range of compatibility with devices.
- <CommunityBadge /> [MemryX](#memryx-mx3): The MX3 Acceleration module is available in m.2 format, offering broad compatibility across various platforms.
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, offering broad compatibility across various platforms.
**AMD**
@ -572,6 +573,132 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht
---
## DEEPX NPU
This detector is available for use with the DEEPX NPU, both the DX-M1 M.2 module and the DX-M1M on the Sixfab AI HAT+ for the Raspberry Pi 5. The configuration below applies unchanged to either form factor. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and passing the NPU through to the container.
To configure a DEEPX detector, set the `type` attribute to `deepx` and follow the configuration guide below.
### Configuration {#configuration-deepx}
<ModelConfigDropdown detectorTitle="DEEPX" models={objectDetectorsModels.deepx.models} />
Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format.
The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of object detection models alongside a JSON file describing how each was compiled: YOLO variants (`model_type: yolo-generic`), SSD (`model_type: ssd`), and DAMO-YOLO (`model_type: damo-yolo`). Pick a model, download the `.dxnn`, bind-mount it into the container, and point `model.path` at it. Note which one you picked, since it determines the `model_type` and `model_format` values below.
`model_type` is set on the detector rather than on the root `model` block. Frigate ignores a `model` block nested under a detector, and the root `model.model_type` defaults to `ssd`, so leaving it unset would silently decode a YOLO model with the SSD decoder and report no detections at all. The detector's own `model_type` defaults to `yolo-generic` and overrides the root value.
Alternatively, compile your own model with the DX-COM compiler.
`model.width` and `model.height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden.
#### Label maps
YOLO and DAMO-YOLO models from the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. SSD models are trained on Pascal VOC's 20-class label set instead, and need `labelmap_path` set to `/labelmap/voc-20.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type for any of these.
#### YOLO model formats {#deepx-yolo-model-formats}
For `model_type: yolo-generic`, DX-COM preserves the detection head of the model it compiles, so the head's tensor layout determines how the output has to be read. Set `model_format` to the layout your `.dxnn` file was compiled with:
```yaml
detectors:
deepx:
type: deepx
model_format: anchor_free
```
Supported values are three decode families:
| `model_format` | Output layout | PPU support |
| -------------- | -------------- | ----------- |
| `anchor` | Anchor-based, with a separate objectness score | Yes |
| `anchor_free` | Anchor-free, no objectness, boxes already in pixels | Yes |
| `nms_in_head` | NMS runs in the detection head, output is final corner boxes | No |
Leaving `model_format` at its `auto` default falls back to inferring the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for anchor-free heads. Setting it explicitly is recommended.
Two mistakes this option prevents:
- Decoding an anchor-free model with the anchor formula squares the box dimensions, producing boxes hundreds of thousands of pixels wide that clip to the entire frame.
- Decoding an anchor-based model as anchor-free reads the objectness column as a class score, so confidences and class IDs are both wrong.
`nms_in_head` models run NMS inside the head, so `nms_threshold` does not apply to them.
#### SSD models
For `model_type: ssd`, no `model_format` is needed. DX-COM keeps SSD's postprocessing inside the compiled model: the `.dxnn` runs the NPU head into a small CPU subgraph that concatenates the per-feature-map outputs, applies the softmax, and decodes the prior-box regression all the way to corner-form boxes. Only NMS is left for the host, so nothing about the backbone has to be named. This applies to all three ModelZoo backbones (MobileNetV1, MobileNetV2-Lite, VGG16) even though they emit different numbers of boxes (3000 and 8732).
```yaml
detectors:
deepx:
type: deepx
model_type: ssd
model:
width: 300
height: 300
labelmap_path: /labelmap/voc-20.txt
```
#### DAMO-YOLO models
For `model_type: damo-yolo`, no `model_format` is needed — all four sizes (TinyNAS-L20T, S, M, L) share one detection head, differing only in backbone depth/width. Leave `model_format` at its `auto` default:
```yaml
detectors:
deepx:
type: deepx
model_type: damo-yolo
model:
width: 640
height: 640
labelmap_path: /labelmap/coco-80.txt
```
#### PPU models
Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. PPU support is currently only available for `model_type: yolo-generic` — there is no confirmed PPU record layout for SSD or DAMO-YOLO yet, and setting `ppu: true` with either is rejected at startup.
Set `ppu: true` on the detector when using a PPU-compiled yolo-generic model:
```yaml
detectors:
deepx:
type: deepx
model_format: anchor_free
ppu: true
score_threshold: 0.25
nms_threshold: 0.45
```
`ppu` and `model_format` are independent: `ppu` says whether candidate selection already happened on the NPU, and `model_format` says how to read the boxes that come back. Both must match how the model was compiled.
`model_format` is required whenever `ppu: true` is set. A PPU record has the same fixed-width layout for every yolo-generic variant, so unlike the raw output path there is no shape to infer the layout from — leaving it at `auto` is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch.
DX-COM cannot compile `nms_in_head` models with PPU support, since their heads already produce final detections. Combining the two is rejected at startup.
#### Multiple NPUs
`device_ids` defaults to `auto`, which binds the detector to every NPU it can find. To pin a detector to specific devices, set it to a comma-separated list of the indices you want it to use. This allows multiple detectors to be configured against separate modules:
```yaml
detectors:
deepx_0:
type: deepx
device_ids: "0"
deepx_1:
type: deepx
device_ids: "1,2"
```
A yaml list (`device_ids: [0, 1]`) is accepted as well, but the comma-separated string is what the config UI writes.
---
## NVidia TensorRT Detector
Nvidia Jetson devices may be used for object detection using the TensorRT libraries. Due to the size of the additional libraries, this detector is only provided in images with the `-tensorrt-jp6` tag suffix, e.g. `ghcr.io/blakeblackshear/frigate:stable-tensorrt-jp6`. This detector is designed to work with Yolo models for object detection.

23
docs/docs/frigate/hardware.md
View File

@ -65,6 +65,11 @@ Frigate supports multiple different detectors that work on different types of ha
- [Supports many model architectures](../../configuration/object_detectors#memryx-mx3)
- Runs best with tiny, small, or medium-size models
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices.
- [Supports YOLO, SSD, and DAMO-YOLO model architectures](../../configuration/object_detectors#deepx-npu)
- Runs best with tiny or small size models
- Runs efficiently on low power hardware
**AMD**
- [ROCm](#rocm---amd-gpu): ROCm can run on AMD Discrete GPUs to provide efficient object detection
@ -256,6 +261,24 @@ The MX3 is a pipelined architecture, where the maximum frames per second support
Inference speeds may vary depending on the host platform. The above data was measured on an **Intel 13700 CPU**. Platforms like Raspberry Pi, Orange Pi, and other ARM-based SBCs have different levels of processing capability, which may limit total FPS.
### DEEPX NPU
Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 module, which works on x86 (Intel/AMD) and ARM-based SBCs such as the Raspberry Pi 5, and the **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) for the Raspberry Pi 5. Both use the same driver and runtime, so the configuration is identical for either one.
DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO, SSD, and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and YOLO models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage.
The DEEPX kernel driver has to be installed on the Docker host before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps.
Detailed information is available [in the detector docs](/configuration/object_detectors#deepx-npu).
| Name | Input Size | Inference Time |
| ------- | ---------- | -------------- |
| YOLOv8s | 640 | ~ 19 ms |
Inference times were measured on a Raspberry Pi 5. The host platform affects the pre- and post-processing cost rather than the inference itself.
### Nvidia Jetson
Jetson devices are supported via the TensorRT or ONNX detectors when running Jetpack 6. It will [make use of the Jetson's hardware media engine](/configuration/hardware_acceleration_video#nvidia-jetson) when configured with the [appropriate presets](/configuration/ffmpeg_presets#hwaccel-presets), and will make use of the Jetson's GPU and DLA for object detection when configured with the [TensorRT detector](/configuration/object_detectors#nvidia-tensorrt-detector).

83
docs/docs/frigate/installation.md
View File

@ -377,6 +377,89 @@ If you can't use Docker Compose, you can run the container with something simila
Finally, configure [hardware object detection](/configuration/object_detectors#memryx-mx3) to complete the setup.
### DEEPX NPU
The DEEPX NPU is available in two form factors, and Frigate supports both:
- **DX-M1** in the M.2 2280 form factor (like an NVMe SSD), for x86 (Intel/AMD) PCs, the Raspberry Pi 5, and other ARM SBCs with an exposed PCIe M.2 slot.
- **DX-M1M** on the [Sixfab AI HAT+](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart), a HAT+ board that connects to the Raspberry Pi 5 over PCIe Gen 3 x1.
Both present the NPU through the same PCIe driver and DX-RT runtime, so the setup below and the detector configuration are identical for either one. Nothing needs to change when moving between them.
DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
#### Versions
A DEEPX install has three separately versioned pieces, and all three have to agree. The DX-RT runtime is built into the Frigate image, so the two host-side pieces must be brought up to match it:
| Component | Version | Installed on | Installed by |
| ------------- | -------- | ------------- | -------------------------- |
| DX-RT runtime | `v3.4.2` | Frigate image | Built in, no action needed |
| Kernel driver | `v2.6.0` | Docker host | `user_installation.sh` |
| NPU firmware | `v2.7.4` | The module | Flashed from the host |
:::warning
A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check all three versions before anything else.
:::
The firmware version cannot be read without the DX-RT runtime installed on the host, so checking or updating it is a manual step done before Frigate is running:
1. Install the DX-RT runtime on the host by following DEEPX's own instructions. If you are using Sixfab hardware, their [AI HAT+ quickstart](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) covers this.
2. Check the reported firmware version and update the module if it does not match the table above.
3. **Remove or disable the host runtime before starting the Frigate container.**
:::danger
Step 3 is not optional when running Frigate in Docker. The DX-RT daemon (`dxrtd`) may only run once per system, and Frigate runs its own copy inside the container. Leaving the host runtime active means two daemons competing for the same NPU, and the container's copy will fail to start.
This only applies to Docker. If you run Frigate directly on the host, it uses the host's runtime and there is no conflict.
:::
#### Installation
The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate.
1. Copy or download [this script](https://github.com/blakeblackshear/frigate/blob/dev/docker/deepx/user_installation.sh).
2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
3. Run the script with `./user_installation.sh`
4. **Restart your computer** to complete driver installation.
Confirm the NPU is visible before continuing:
```bash
ls /dev/dxrt*
```
:::note
The installation script disables the host's `dxrt.service` if it finds one, for the reason described above. If you install or reinstall the DEEPX runtime on the host afterwards, disable it again before starting the container:
```bash
sudo systemctl disable --now dxrt.service
```
:::
#### Setup
To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
Next, grant Docker access to the NPU by adding the device to your `docker-compose.yml` file:
```yaml
devices:
- /dev/dxrt0
```
If you can't use Docker Compose, add `--device /dev/dxrt0` to your `docker run` command.
#### Configuration
Finally, configure [hardware object detection](/configuration/object_detectors#deepx-npu) to complete the setup.
### Rockchip platform
Make sure that you use a linux distribution that comes with the rockchip BSP kernel 5.10 or 6.1 and necessary drivers (especially rkvdec2 and rknpu). To check, enter the following commands:

3
frigate/detectors/detector_config.py
View File

@ -36,6 +36,7 @@ class InputDTypeEnum(str, Enum):
class ModelTypeEnum(str, Enum):
damoyolo = "damo-yolo"
dfine = "dfine"
rfdetr = "rfdetr"
ssd = "ssd"
@ -93,7 +94,7 @@ class ModelConfig(BaseModel):
model_type: ModelTypeEnum = Field(
default=ModelTypeEnum.ssd,
title="Object Detection Model Type",
description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization.",
description="Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization.",
)
_merged_labelmap: dict[int, str] | None = PrivateAttr()
_colormap: dict[int, tuple[int, int, int]] = PrivateAttr()

1143
frigate/detectors/plugins/deepx.py Normal file
View File

File diff suppressed because it is too large Load Diff

757
frigate/test/test_deepx.py Normal file
View File

@ -0,0 +1,757 @@
"""Tests for the DEEPX detector's per-format output decoding."""
import unittest
import numpy as np
from pydantic import ValidationError
from frigate.detectors.detector_config import ModelTypeEnum
from frigate.detectors.plugins.deepx import (
ANCHOR_FORMATS,
ANCHOR_FREE_FORMATS,
DEVICE_IDS_AUTO,
NMS_IN_HEAD_FORMATS,
PPU_ANCHORS,
PPU_RECORD_SIZE,
DeepxDetectorConfig,
DeepxModelTypeEnum,
ModelFormatEnum,
decode_damoyolo_raw,
decode_ppu_anchor,
decode_ppu_anchor_free,
decode_raw_anchor,
decode_raw_anchor_free,
decode_raw_nms_in_head,
decode_ssd_raw,
parse_device_ids,
resolve_devices,
)
def build_ppu_record(
box: tuple[float, float, float, float],
grid_y: int,
grid_x: int,
anchor_idx: int,
layer_idx: int,
score: float,
label: int,
) -> np.ndarray:
"""Build a single fixed-width detection record as the PPU emits it."""
record = np.zeros(PPU_RECORD_SIZE, dtype=np.uint8)
record[0:16] = np.array(box, dtype=np.float32).view(np.uint8)
record[16:20] = [grid_y, grid_x, anchor_idx, layer_idx]
record[20:24] = np.array([score], dtype=np.float32).view(np.uint8)
record[24:28] = np.array([label], dtype=np.uint32).view(np.uint8)
return record
class TestDeepxPpuDecode(unittest.TestCase):
def test_decodes_a_single_detection(self):
# stride 8 (layer 0) with the first anchor of that layer
outputs = [
build_ppu_record(
box=(0.5, 0.5, 0.5, 0.5),
grid_y=10,
grid_x=10,
anchor_idx=0,
layer_idx=0,
score=0.9,
label=2,
).reshape(1, 1, PPU_RECORD_SIZE)
]
detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
anchor_w, anchor_h = PPU_ANCHORS[8][0]
# center = (0.5 * 2 - 0.5 + 10) * 8, size = (0.5 ** 2 * 4) * anchor
center = (0.5 * 2.0 - 0.5 + 10) * 8
box_w = 0.5**2 * 4.0 * anchor_w
box_h = 0.5**2 * 4.0 * anchor_h
self.assertEqual(detections[0][0], 2)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertAlmostEqual(detections[0][2], (center - box_h / 2) / 640, places=5)
self.assertAlmostEqual(detections[0][3], (center - box_w / 2) / 640, places=5)
self.assertAlmostEqual(detections[0][4], (center + box_h / 2) / 640, places=5)
self.assertAlmostEqual(detections[0][5], (center + box_w / 2) / 640, places=5)
# remaining slots stay empty
self.assertTrue(np.all(detections[1:] == 0))
def test_uses_the_anchor_for_the_records_layer(self):
# the same box on stride 32 (layer 2) must decode to a larger box
def decode(layer_idx: int) -> np.ndarray:
outputs = [
build_ppu_record(
box=(0.5, 0.5, 0.5, 0.5),
grid_y=1,
grid_x=1,
anchor_idx=0,
layer_idx=layer_idx,
score=0.9,
label=0,
).reshape(1, 1, PPU_RECORD_SIZE)
]
return decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
small = decode(0)
large = decode(2)
small_width = small[0][5] - small[0][3]
large_width = large[0][5] - large[0][3]
self.assertGreater(large_width, small_width)
def test_drops_detections_below_the_score_threshold(self):
outputs = [
build_ppu_record(
box=(0.5, 0.5, 0.5, 0.5),
grid_y=10,
grid_x=10,
anchor_idx=0,
layer_idx=0,
score=0.1,
label=2,
).reshape(1, 1, PPU_RECORD_SIZE)
]
detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
def test_caps_output_at_twenty_detections(self):
records = np.stack(
[
build_ppu_record(
box=(0.5, 0.5, 0.5, 0.5),
grid_y=i,
grid_x=i,
anchor_idx=0,
layer_idx=0,
score=0.9,
label=1,
)
for i in range(40)
]
)
detections = decode_ppu_anchor(
[records.reshape(1, 40, PPU_RECORD_SIZE)], 640, 640, 0.25, 0.45
)
self.assertEqual(detections.shape, (20, 6))
def test_returns_empty_detections_for_no_output(self):
self.assertTrue(np.all(decode_ppu_anchor([], 640, 640, 0.25, 0.45) == 0))
def test_returns_empty_detections_for_an_unexpected_record_width(self):
# a non-PPU model would produce a different record width
outputs = [np.zeros((1, 4, 16), dtype=np.uint8)]
self.assertTrue(np.all(decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) == 0))
class TestDeepxModelFormats(unittest.TestCase):
def test_every_yolo_format_belongs_to_exactly_one_family(self):
families = [ANCHOR_FORMATS, ANCHOR_FREE_FORMATS, NMS_IN_HEAD_FORMATS]
yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS
for fmt in ModelFormatEnum:
if fmt not in yolo_formats:
continue
matches = [family for family in families if fmt in family]
self.assertEqual(
len(matches), 1, f"{fmt.value} must be in exactly one family"
)
def test_families_do_not_overlap(self):
self.assertFalse(ANCHOR_FORMATS & ANCHOR_FREE_FORMATS)
self.assertFalse(ANCHOR_FORMATS & NMS_IN_HEAD_FORMATS)
self.assertFalse(ANCHOR_FREE_FORMATS & NMS_IN_HEAD_FORMATS)
def test_every_format_belongs_to_a_yolo_family(self):
# apart from the `auto` sentinel, every ModelFormatEnum member is a
# yolo-generic head shape -- nothing should be unclassified
yolo_formats = ANCHOR_FORMATS | ANCHOR_FREE_FORMATS | NMS_IN_HEAD_FORMATS
for fmt in ModelFormatEnum:
if fmt is ModelFormatEnum.auto:
continue
self.assertIn(fmt, yolo_formats, f"{fmt.value} is not classified")
def test_auto_is_in_no_family(self):
"""`auto` means "no explicit layout", so every decode branch keyed off
a family must fall through to shape-based inference."""
self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FORMATS)
self.assertNotIn(ModelFormatEnum.auto, ANCHOR_FREE_FORMATS)
self.assertNotIn(ModelFormatEnum.auto, NMS_IN_HEAD_FORMATS)
class TestDeepxDetectorConfig(unittest.TestCase):
def test_ppu_without_model_format_is_rejected(self):
"""PPU records are fixed-width regardless of variant, so an unset
model_format can't be inferred and must not silently fall back to the
anchor decoder."""
with self.assertRaises(ValidationError):
DeepxDetectorConfig(type="deepx", ppu=True)
def test_ppu_with_anchor_free_model_format_is_accepted(self):
config = DeepxDetectorConfig(
type="deepx", ppu=True, model_format=ModelFormatEnum.anchor_free
)
self.assertEqual(config.model_format, ModelFormatEnum.anchor_free)
def test_raw_without_model_format_is_still_accepted(self):
# the raw path falls back to shape-based inference, so this remains
# optional when ppu is not set
config = DeepxDetectorConfig(type="deepx")
self.assertIs(config.model_format, ModelFormatEnum.auto)
def test_an_omitted_model_format_folds_to_auto(self):
"""The config form round-trips a value for every field, so a blank or
null entry has to land on the sentinel rather than stay None."""
for value in (None, "", " "):
with self.subTest(value=value):
config = DeepxDetectorConfig(type="deepx", model_format=value)
self.assertIs(config.model_format, ModelFormatEnum.auto)
def test_ppu_with_damoyolo_model_type_is_rejected(self):
"""No confirmed PPU record layout exists for DAMO-YOLO yet."""
with self.assertRaises(ValidationError):
DeepxDetectorConfig(
type="deepx",
ppu=True,
model_format=ModelFormatEnum.anchor_free,
model_type=DeepxModelTypeEnum.damoyolo,
)
def test_ssd_model_type_takes_no_model_format(self):
"""SSD's layout is fixed: DX-COM keeps the softmax and prior decode
inside the .dxnn, so no backbone has to be named."""
config = DeepxDetectorConfig(type="deepx", model_type=DeepxModelTypeEnum.ssd)
self.assertIs(config.model_format, ModelFormatEnum.auto)
def test_ssd_model_type_rejects_a_model_format(self):
with self.assertRaises(ValidationError):
DeepxDetectorConfig(
type="deepx",
model_format=ModelFormatEnum.anchor_free,
model_type=DeepxModelTypeEnum.ssd,
)
def test_damoyolo_model_type_rejects_a_model_format(self):
with self.assertRaises(ValidationError):
DeepxDetectorConfig(
type="deepx",
model_format=ModelFormatEnum.anchor_free,
model_type=DeepxModelTypeEnum.damoyolo,
)
def test_damoyolo_model_type_without_model_format_is_accepted(self):
config = DeepxDetectorConfig(
type="deepx",
model_type=DeepxModelTypeEnum.damoyolo,
)
self.assertIs(config.model_format, ModelFormatEnum.auto)
class TestDeepxDeviceIds(unittest.TestCase):
def test_a_comma_separated_string_is_kept(self):
config = DeepxDetectorConfig(type="deepx", device_ids="0,1")
self.assertEqual(config.device_ids, "0,1")
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
def test_a_yaml_list_is_accepted(self):
"""A bare list is the natural thing to hand-write, so it is folded
into the same comma-separated form the config UI writes."""
config = DeepxDetectorConfig(type="deepx", device_ids=[0, 1])
self.assertEqual(config.device_ids, "0,1")
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
def test_a_single_index_is_accepted(self):
config = DeepxDetectorConfig(type="deepx", device_ids=1)
self.assertEqual(config.device_ids, "1")
self.assertEqual(resolve_devices(config.device_ids), [1])
def test_surrounding_whitespace_is_tolerated(self):
config = DeepxDetectorConfig(type="deepx", device_ids=" 0 , 1 ")
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
def test_an_unparsable_value_is_rejected_at_config_load(self):
with self.assertRaises(ValidationError):
DeepxDetectorConfig(type="deepx", device_ids="gpu0")
def test_unset_leaves_device_selection_to_auto_detection(self):
config = DeepxDetectorConfig(type="deepx")
self.assertEqual(config.device_ids, DEVICE_IDS_AUTO)
self.assertEqual(parse_device_ids(config.device_ids), [])
def test_a_blank_entry_folds_to_auto(self):
"""The config form round-trips a value for every field, so a blank or
null entry has to land on the sentinel rather than stay None."""
for value in (None, "", " ", []):
with self.subTest(value=value):
config = DeepxDetectorConfig(type="deepx", device_ids=value)
self.assertEqual(config.device_ids, DEVICE_IDS_AUTO)
def test_auto_is_case_insensitive(self):
config = DeepxDetectorConfig(type="deepx", device_ids="AUTO")
self.assertEqual(parse_device_ids(config.device_ids), [])
class TestDeepxPpuAnchorFreeDecode(unittest.TestCase):
def test_reads_the_box_bytes_as_pixel_geometry(self):
"""An anchor-free PPU record holds cx, cy, w, h directly in pixels."""
record = build_ppu_record(
box=(320.0, 160.0, 64.0, 32.0),
# grid columns carry no meaning for an anchor-free head, and must
# not influence the result
grid_y=7,
grid_x=9,
anchor_idx=2,
layer_idx=2,
score=0.9,
label=0,
)
outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)]
detections = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45)
# x: 320 +/- 32 -> 288..352, y: 160 +/- 16 -> 144..176
self.assertAlmostEqual(detections[0][2], 144 / 640, places=5)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
self.assertAlmostEqual(detections[0][4], 176 / 640, places=5)
self.assertAlmostEqual(detections[0][5], 352 / 640, places=5)
def test_anchor_formula_would_blow_the_box_up(self):
"""Guards the bug this decoder exists to prevent."""
record = build_ppu_record(
box=(320.0, 160.0, 64.0, 32.0),
grid_y=7,
grid_x=9,
anchor_idx=2,
layer_idx=2,
score=0.9,
label=0,
)
outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)]
anchor_free = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45)
anchor = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
# the anchor decode squares the width, saturating the clip to the frame
self.assertEqual(anchor[0][5], 1.0)
self.assertLess(anchor_free[0][5], 1.0)
class TestDeepxRawAnchorDecode(unittest.TestCase):
def build_raw_anchor_output(self, rows):
"""Build an (1, N, 5+C) tensor with 80 classes."""
out = np.zeros((1, len(rows), 85), dtype=np.float32)
for i, (cx, cy, w, h, obj, label, cls_score) in enumerate(rows):
out[0, i, 0:4] = [cx, cy, w, h]
out[0, i, 4] = obj
out[0, i, 5 + label] = cls_score
return [out]
def test_confidence_is_objectness_times_class_score(self):
outputs = self.build_raw_anchor_output(
[(320.0, 320.0, 40.0, 80.0, 0.8, 3, 0.5)]
)
detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45)
self.assertEqual(detections[0][0], 3)
self.assertAlmostEqual(detections[0][1], 0.4, places=5)
def test_drops_rows_whose_combined_score_is_below_threshold(self):
# 0.4 * 0.5 = 0.2, under a 0.25 threshold even though both parts are
# individually above it
outputs = self.build_raw_anchor_output(
[(320.0, 320.0, 40.0, 80.0, 0.4, 3, 0.5)]
)
detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
def test_converts_center_form_to_normalized_corners(self):
outputs = self.build_raw_anchor_output(
[(320.0, 160.0, 64.0, 32.0, 1.0, 0, 1.0)]
)
detections = decode_raw_anchor(outputs, 640, 640, 0.25, 0.45)
self.assertAlmostEqual(detections[0][2], 144 / 640, places=5)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
self.assertAlmostEqual(detections[0][4], 176 / 640, places=5)
self.assertAlmostEqual(detections[0][5], 352 / 640, places=5)
class TestDeepxRawAnchorFreeDecode(unittest.TestCase):
def build_raw_anchor_free_output(self, rows, num_classes=3, total_rows=10):
"""Build an (1, N, 4+C) tensor -- already row-per-box -- with no
objectness column, padded with all-zero rows so N comfortably
outnumbers the channel count (4+C).
The decoder tells a channel-major export apart from a row-major one
purely by comparing those two sizes, the same way it would on a real
grid of thousands of anchors -- and that comparison only comes out
right when boxes outnumber channels, so a test with too few rows
would trip the same ambiguity `auto` accepts by design.
"""
total_rows = max(total_rows, len(rows))
out = np.zeros((1, total_rows, 4 + num_classes), dtype=np.float32)
for i, (cx, cy, w, h, label, cls_score) in enumerate(rows):
out[0, i, 0:4] = [cx, cy, w, h]
out[0, i, 4 + label] = cls_score
return [out]
def test_keeps_a_detection_between_the_deepx_and_frigate_default_thresholds(self):
"""Regression test: post_process_yolo hardcodes a 0.4 score
threshold, which would silently drop this 0.3-confidence detection
even though it clears the DEEPX decoder's own 0.25 default."""
outputs = self.build_raw_anchor_free_output(
[(320.0, 320.0, 40.0, 80.0, 2, 0.3)]
)
detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45)
self.assertEqual(detections[0][0], 2)
self.assertAlmostEqual(detections[0][1], 0.3, places=5)
def test_drops_detections_below_the_score_threshold(self):
outputs = self.build_raw_anchor_free_output(
[(320.0, 320.0, 40.0, 80.0, 2, 0.1)]
)
detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
def test_transposes_a_channel_major_output(self):
"""DX-COM may export (4+C, N) instead of (N, 4+C); both must decode
to the same result."""
row_major = self.build_raw_anchor_free_output(
[(320.0, 320.0, 40.0, 80.0, 2, 0.9)]
)
channel_major = [np.transpose(row_major[0], (0, 2, 1))]
row_major_detections = decode_raw_anchor_free(row_major, 640, 640, 0.25, 0.45)
channel_major_detections = decode_raw_anchor_free(
channel_major, 640, 640, 0.25, 0.45
)
np.testing.assert_array_almost_equal(
row_major_detections, channel_major_detections
)
def test_converts_center_form_to_normalized_corners(self):
outputs = self.build_raw_anchor_free_output(
[(320.0, 160.0, 64.0, 32.0, 0, 1.0)]
)
detections = decode_raw_anchor_free(outputs, 640, 640, 0.25, 0.45)
self.assertAlmostEqual(detections[0][2], 144 / 640, places=5)
self.assertAlmostEqual(detections[0][3], 288 / 640, places=5)
self.assertAlmostEqual(detections[0][4], 176 / 640, places=5)
self.assertAlmostEqual(detections[0][5], 352 / 640, places=5)
def test_returns_empty_detections_for_no_output(self):
out = np.zeros((1, 0, 84), dtype=np.float32)
self.assertTrue(np.all(decode_raw_anchor_free([out], 640, 640, 0.25, 0.45) == 0))
class TestDeepxRawNmsInHeadDecode(unittest.TestCase):
def test_reads_corner_records_without_running_nms(self):
"""Two heavily overlapping boxes both survive: the head already ran NMS."""
out = np.array(
[
[
[100.0, 100.0, 200.0, 200.0, 0.9, 2.0],
[102.0, 102.0, 202.0, 202.0, 0.8, 2.0],
]
],
dtype=np.float32,
)
detections = decode_raw_nms_in_head([out], 640, 640, 0.25)
self.assertEqual(detections[0][0], 2)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertAlmostEqual(detections[0][3], 100 / 640, places=5)
self.assertEqual(detections[1][0], 2)
self.assertAlmostEqual(detections[1][1], 0.8, places=5)
def test_applies_the_score_threshold(self):
out = np.array(
[
[
[100.0, 100.0, 200.0, 200.0, 0.9, 2.0],
[300.0, 300.0, 400.0, 400.0, 0.1, 5.0],
]
],
dtype=np.float32,
)
detections = decode_raw_nms_in_head([out], 640, 640, 0.25)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertTrue(np.all(detections[1] == 0))
def test_returns_empty_detections_for_an_empty_output(self):
out = np.zeros((1, 0, 6), dtype=np.float32)
self.assertTrue(np.all(decode_raw_nms_in_head([out], 640, 640, 0.25) == 0))
class TestDeepxSsdDecode(unittest.TestCase):
"""The .dxnn emits final boxes and probabilities, so these feed the
decoder what a real compiled SSD produces: corner-form boxes normalized
to [0, 1] and post-softmax confidences with background at column 0."""
def build_ssd_output(self, box_index, box, class_id, prob, num_boxes=3000):
num_classes = 21 # 20 VOC classes + background
boxes = np.zeros((1, num_boxes, 4), dtype=np.float32)
boxes[0, box_index] = box
# every other box is background, as the model would report it
scores = np.zeros((1, num_boxes, num_classes), dtype=np.float32)
scores[0, :, 0] = 1.0
scores[0, box_index, 0] = 1.0 - prob
scores[0, box_index, class_id] = prob
return [boxes, scores]
def test_decodes_an_already_decoded_box(self):
outputs = self.build_ssd_output(
box_index=7, box=(0.1, 0.2, 0.5, 0.8), class_id=5, prob=0.9
)
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
# boxes pass through untouched apart from the corner reorder, and
# the background class is dropped so labels shift down by one
self.assertEqual(detections[0][0], 4)
self.assertAlmostEqual(float(detections[0][1]), 0.9, places=5)
self.assertAlmostEqual(float(detections[0][2]), 0.2, places=5) # y_min
self.assertAlmostEqual(float(detections[0][3]), 0.1, places=5) # x_min
self.assertAlmostEqual(float(detections[0][4]), 0.8, places=5) # y_max
self.assertAlmostEqual(float(detections[0][5]), 0.5, places=5) # x_max
def test_confidences_are_not_softmaxed_again(self):
"""A second softmax over 21 already-normalized probabilities pulls
every score toward 1/21, which is what silently dropped every real
detection under the default threshold."""
outputs = self.build_ssd_output(
box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=15, prob=0.6
)
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
self.assertAlmostEqual(float(detections[0][1]), 0.6, places=5)
def test_any_box_count_is_accepted(self):
"""3000 for the MobileNet backbones, 8732 for VGG16 -- the decoder
reads whatever the model emits rather than a per-backbone table."""
for num_boxes in (3000, 8732):
with self.subTest(num_boxes=num_boxes):
outputs = self.build_ssd_output(
box_index=num_boxes - 1,
box=(0.0, 0.0, 1.0, 1.0),
class_id=1,
prob=0.8,
num_boxes=num_boxes,
)
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
self.assertEqual(detections[0][0], 0)
self.assertAlmostEqual(float(detections[0][1]), 0.8, places=5)
def test_output_order_does_not_matter(self):
boxes, scores = self.build_ssd_output(
box_index=1, box=(0.2, 0.2, 0.6, 0.6), class_id=3, prob=0.7
)
forward = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45)
reversed_ = decode_ssd_raw([scores, boxes], 300, 300, 0.25, 0.45)
np.testing.assert_array_equal(forward, reversed_)
def test_low_confidence_detection_is_dropped(self):
outputs = self.build_ssd_output(
box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=3, prob=0.05
)
detections = decode_ssd_raw(outputs, 300, 300, 0.5, 0.45)
self.assertTrue(np.all(detections == 0))
def test_returns_empty_detections_when_outputs_cannot_be_told_apart(self):
# two tensors that both look class-shaped (last dim != 4)
outputs = [
np.zeros((1, 10, 21), dtype=np.float32),
np.zeros((1, 10, 21), dtype=np.float32),
]
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
def test_returns_empty_detections_when_box_and_class_counts_disagree(self):
boxes = np.zeros((1, 10, 4), dtype=np.float32)
scores = np.zeros((1, 12, 21), dtype=np.float32)
detections = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
class TestDeepxDamoyoloDecode(unittest.TestCase):
def total_priors(self, width, height, strides):
return sum((height // s) * (width // s) for s in strides)
def test_decodes_a_detection_from_already_decoded_boxes(self):
"""box_output shaped (1, N, 4): DX-COM's export is assumed to have
already applied the full distance-to-box decode, so these are final
(x_min, y_min, x_max, y_max) pixel coordinates, used as-is."""
width = height = 640
num_priors = 100
prior_index = 5
cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32)
cls_scores[0, prior_index, 7] = 0.9
box_output = np.zeros((1, num_priors, 4), dtype=np.float32)
box_output[0, prior_index] = [100.0, 120.0, 140.0, 160.0]
detections = decode_damoyolo_raw(
[cls_scores, box_output], width, height, 0.25, 0.45
)
self.assertEqual(detections[0][0], 7)
self.assertAlmostEqual(detections[0][1], 0.9, places=5)
self.assertAlmostEqual(detections[0][2], 120 / height, places=5)
self.assertAlmostEqual(detections[0][3], 100 / width, places=5)
self.assertAlmostEqual(detections[0][4], 160 / height, places=5)
self.assertAlmostEqual(detections[0][5], 140 / width, places=5)
def test_decodes_a_detection_from_a_dfl_distribution(self):
"""box_output shaped (1, N, 4, bins): a raw per-side DFL histogram,
needing softmax + integral + stride-scaled distance-to-box here."""
width = height = 640
strides = (8, 16, 32)
grid_w = width // 8
prior_row, prior_col = 10, 10
prior_index = prior_row * grid_w + prior_col
num_priors = self.total_priors(width, height, strides)
cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32)
cls_scores[0, prior_index, 9] = 0.8
# reg_max=1 (2 bins): put ~all softmax mass on bin index 1, so the
# integral (weighted sum against [0, 1]) comes out to ~1.0 stride
# unit on every side
box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32)
box_output[0, prior_index, :, 0] = -30.0
box_output[0, prior_index, :, 1] = 30.0
detections = decode_damoyolo_raw(
[cls_scores, box_output], width, height, 0.25, 0.45, strides
)
# stride-8 grid cell (10, 10) centers at pixel (80, 80); ~1
# stride-unit distance on every side -> an 8px box around it
self.assertEqual(detections[0][0], 9)
self.assertAlmostEqual(detections[0][1], 0.8, places=5)
self.assertAlmostEqual(detections[0][2], 72 / height, places=3)
self.assertAlmostEqual(detections[0][3], 72 / width, places=3)
self.assertAlmostEqual(detections[0][4], 88 / height, places=3)
self.assertAlmostEqual(detections[0][5], 88 / width, places=3)
def test_returns_empty_detections_below_score_threshold(self):
num_priors = 10
cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32)
cls_scores[0, 0, 1] = 0.1
box_output = np.zeros((1, num_priors, 4), dtype=np.float32)
box_output[0, 0] = [10.0, 10.0, 20.0, 20.0]
detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.5, 0.45)
self.assertTrue(np.all(detections == 0))
def test_returns_empty_detections_when_dfl_prior_count_does_not_match_the_grid(
self,
):
# a DFL-shaped box output with far fewer priors than the (8, 16, 32)
# strides actually produce for a 640x640 input
num_priors = 5
cls_scores = np.zeros((1, num_priors, 80), dtype=np.float32)
cls_scores[0, 0, 1] = 0.9
box_output = np.zeros((1, num_priors, 4, 2), dtype=np.float32)
detections = decode_damoyolo_raw(
[cls_scores, box_output], 640, 640, 0.25, 0.45, (8, 16, 32)
)
self.assertTrue(np.all(detections == 0))
def test_returns_empty_detections_when_box_and_class_counts_disagree(self):
cls_scores = np.zeros((1, 10, 80), dtype=np.float32)
cls_scores[0, 0, 1] = 0.9
box_output = np.zeros((1, 7, 4), dtype=np.float32)
detections = decode_damoyolo_raw([cls_scores, box_output], 640, 640, 0.25, 0.45)
self.assertTrue(np.all(detections == 0))
def test_returns_empty_detections_for_wrong_output_count(self):
outputs = [np.zeros((1, 10, 80), dtype=np.float32)]
detections = decode_damoyolo_raw(outputs, 640, 640, 0.25, 0.45, (8, 16, 32))
self.assertTrue(np.all(detections == 0))
if __name__ == "__main__":
unittest.main()
class TestDeepxModelType(unittest.TestCase):
def test_model_type_defaults_to_yolo_generic(self):
"""Frigate's own model_type default is ssd, which silently picked the
SSD decoder for anyone who left it unset."""
config = DeepxDetectorConfig(type="deepx")
self.assertIs(config.model_type, DeepxModelTypeEnum.yologeneric)
def test_model_type_values_match_frigate_model_types(self):
"""The detector writes its choice back onto model.model_type, so every
value has to round-trip through ModelTypeEnum."""
for value in DeepxModelTypeEnum:
with self.subTest(value=value):
self.assertEqual(ModelTypeEnum(value.value).value, value.value)

30
web/public/locales/en/config/global.json
View File

@ -322,7 +322,7 @@
},
"model_type": {
"label": "Object Detection Model Type",
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization."
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization."
}
},
"model_path": {
@ -357,6 +357,32 @@
"description": "Optional API key for authenticated DeepStack services."
}
},
"deepx": {
"label": "DEEPX NPU",
"description": "DEEPX NPU detector running .dxnn models via the DX-RT runtime.",
"device_ids": {
"label": "DEEPX device indices",
"description": "Comma-separated NPU device indices this detector should bind to, e.g. \"0,1\". Leave as \"auto\" to detect them from the DXNN_DEVICES environment variable, then the DX-RT device count, then a scan of /dev/dxrt*."
},
"model_type": {
"label": "Detection model architecture",
"description": "Which decoder reads the model's output. Overrides model.model_type for this detector, whose default of ssd would otherwise apply whenever the root model block leaves it unset."
},
"ppu": {
"label": "PPU-compiled model",
"description": "Set to true when the model was compiled with DEEPX's Post-Processing Unit support, which moves candidate selection onto the NPU and changes the output format. Only supported for model_type yolo-generic."
},
"model_format": {
"label": "Detection head layout the model was compiled from",
"description": "Selects how the model's output is decoded. Only used for model_type yolo-generic: leave as \"auto\" to infer the layout from the output shape, which cannot tell an anchor-based head from an anchor-free one and is only reliable for anchor-free heads. Required when ppu is true, since a PPU record's layout is identical for every yolo-generic variant and cannot be inferred. ssd and damo-yolo each read one fixed layout and take no value here."
},
"score_threshold": {
"label": "Score threshold used when decoding detector output"
},
"nms_threshold": {
"label": "NMS IoU threshold used when decoding detector output"
}
},
"edgetpu": {
"label": "EdgeTPU",
"description": "EdgeTPU detector that runs TensorFlow Lite models compiled for Coral EdgeTPU using the EdgeTPU delegate.",
@ -479,7 +505,7 @@
},
"model_type": {
"label": "Object Detection Model Type",
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization."
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization."
}
},
"genai": {