mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-10-18 07:29:45 +00:00
Add tests and updates for DEEPX detector and refactor DEEPX accelerator code structure.
This commit is contained in:
aahmetdurmaz
parent
cbf465f1b4
commit
cca4a3b046
@ -1,44 +1,25 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Installs the DEEPX NPU kernel driver on the Docker host.
|
||||
# Installs the DEEPX NPU kernel driver and the DX-RT runtime on the Docker host,
|
||||
# then enables the vendor's dxrt.service. A container cannot load kernel
|
||||
# modules, so this runs outside the image; the driver creates the /dev/dxrt*
|
||||
# nodes and the daemon multiplexes the NPU across host and container.
|
||||
#
|
||||
# The driver has to be built and loaded on the host rather than in the Frigate
|
||||
# container: containers share the host kernel, so a kernel module cannot be
|
||||
# loaded from inside one. Once loaded it exposes the NPU as /dev/dxrt* device
|
||||
# nodes, which are passed through to the container (see the DEEPX section of
|
||||
# the installation docs).
|
||||
# Driver, runtime (v3.4.0) and NPU firmware (v2.7.4) must agree or inference
|
||||
# hangs instead of failing at startup; 3.4.0 needs driver 2.5.0, firmware 2.7.0.
|
||||
#
|
||||
# A DEEPX install has three separately versioned pieces that have to agree:
|
||||
#
|
||||
# DX-RT runtime v3.4.2 the Frigate image, docker/main/install_dxrt.sh
|
||||
# Kernel driver v2.6.0 this file (provides dxrt_driver and dx_dma)
|
||||
# NPU firmware v2.7.4 the module itself, flashed from the host
|
||||
#
|
||||
# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the
|
||||
# versions pinned here sit above that floor. A mismatch is the most common
|
||||
# cause of inference requests that are accepted but never complete, and it
|
||||
# fails silently rather than at startup. This script only installs the driver:
|
||||
# reading or flashing the firmware version requires the DX-RT runtime on the
|
||||
# host, which conflicts with the copy Frigate runs in its container, so that is
|
||||
# a separate manual step covered in the installation docs.
|
||||
#
|
||||
# DEEPX NPU support in Frigate is developed and maintained by Sixfab
|
||||
# (https://sixfab.com).
|
||||
# DEEPX NPU support in Frigate is maintained by Sixfab (https://sixfab.com).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
driver_version="v2.6.0"
|
||||
# Commit the tag above resolves to, pinned so a later retag or repoint of
|
||||
# v2.6.0 upstream can't silently swap in different source between review and
|
||||
# build, before it's compiled and installed as root. DEEPX doesn't sign tags
|
||||
# or publish checksums for this repo, so an immutable commit is the strongest
|
||||
# verification available. Update both together when bumping driver_version:
|
||||
# git ls-remote https://github.com/DEEPX-AI/dx_rt_npu_linux_driver.git "refs/tags/<version>"
|
||||
# the commit the tag resolves to, since DEEPX signs neither tags nor releases
|
||||
# and this is compiled and installed as root. Update both together
|
||||
driver_commit="7074748e7104f470b02f517583abba652b3f05fa"
|
||||
firmware_version="v2.7.4"
|
||||
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils
|
||||
sudo apt-get install -y git build-essential "linux-headers-$(uname -r)" pciutils wget
|
||||
|
||||
if ! lspci -d 1ff4: | grep -q .; then
|
||||
echo "No DEEPX device found on the PCIe bus (lspci -d 1ff4:)."
|
||||
@ -46,10 +27,8 @@ if ! lspci -d 1ff4: | grep -q .; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Fetch the pinned commit directly (GitHub allows fetching a public repo by
|
||||
# SHA, not just by branch/tag) rather than cloning "${driver_version}", so
|
||||
# driver_commit above is what actually determines the source, not a ref that
|
||||
# could move after this script was reviewed.
|
||||
# fetch the pinned commit rather than cloning the tag, so a retag cannot swap
|
||||
# in different source
|
||||
mkdir dx_rt_npu_linux_driver
|
||||
cd dx_rt_npu_linux_driver
|
||||
git init -q
|
||||
@ -66,13 +45,11 @@ fi
|
||||
|
||||
cd modules
|
||||
|
||||
# --reload unloads any running modules and loads the newly built ones, so the
|
||||
# NPU is usable without a reboot.
|
||||
sudo ./build.sh -c install --reload
|
||||
|
||||
sudo depmod -A
|
||||
|
||||
# dx_dma is the PCIe transport, dxrt_driver is the NPU driver on top of it.
|
||||
# dx_dma is the PCIe transport, dxrt_driver the NPU driver on top of it
|
||||
for module in dx_dma dxrt_driver; do
|
||||
if ! sudo modprobe "${module}"; then
|
||||
echo "Unable to load the ${module} kernel module, common reasons are:"
|
||||
@ -88,18 +65,47 @@ if ! compgen -G "/dev/dxrt*" > /dev/null; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# dxrtd runs inside the Frigate container and may only run once per system, so
|
||||
# a host copy left over from a runtime install has to be disabled.
|
||||
if systemctl list-unit-files dxrt.service &> /dev/null; then
|
||||
echo "Disabling the host dxrt.service so the container can run dxrtd."
|
||||
sudo systemctl disable --now dxrt.service
|
||||
runtime_version="v3.4.0"
|
||||
declare -A runtime_sha256=(
|
||||
[amd64]="736cfef009ce9e974ab1ab610d867239d19d72a426a53e367ddcbd53297b6e20"
|
||||
[arm64]="eb6107f5f02f2ad76ae89f414e8b5f346f34fbc6f0888236136853a26be6f6a0"
|
||||
)
|
||||
|
||||
runtime_release="${runtime_version#v}"
|
||||
deb_arch=$(dpkg --print-architecture)
|
||||
deb_file="/tmp/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
|
||||
|
||||
wget -qO "${deb_file}" \
|
||||
"https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${runtime_version}/release/${runtime_release}/libdxrt-bin_${runtime_release}_${deb_arch}.deb"
|
||||
|
||||
expected_sha256="${runtime_sha256[${deb_arch}]:-}"
|
||||
if [[ -z "${expected_sha256}" ]]; then
|
||||
echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install."
|
||||
exit 1
|
||||
fi
|
||||
if [[ "$(sha256sum "${deb_file}" | cut -d' ' -f1)" != "${expected_sha256}" ]]; then
|
||||
echo "SHA-256 mismatch for ${deb_file}; refusing to install."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "DEEPX driver installation complete."
|
||||
echo "Driver version: $(modinfo -F version dxrt_driver)"
|
||||
echo "Device node(s): $(echo /dev/dxrt*)"
|
||||
sudo dpkg -i "${deb_file}"
|
||||
sudo ldconfig
|
||||
rm -f "${deb_file}"
|
||||
|
||||
sudo cp /usr/share/libdxrt-bin/service/dxrt.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now dxrt.service
|
||||
|
||||
if ! sudo systemctl is-active --quiet dxrt.service; then
|
||||
echo "dxrt.service did not start. Check: sudo journalctl -u dxrt.service"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "DEEPX driver and runtime installation complete."
|
||||
echo "Driver version: $(modinfo -F version dxrt_driver) (expected ${driver_version#v})"
|
||||
echo "Runtime version: ${runtime_release}"
|
||||
echo "Device node(s): $(echo /dev/dxrt*)"
|
||||
echo
|
||||
echo "This driver expects NPU firmware ${firmware_version}. The firmware version"
|
||||
echo "can only be read with the DX-RT runtime installed on the host, so check it"
|
||||
echo "before starting Frigate, then remove the host runtime: Frigate runs its own"
|
||||
echo "copy of dxrtd and only one may run per system."
|
||||
echo "This driver expects NPU firmware ${firmware_version}. Check it with:"
|
||||
echo " dxrt-cli --status"
|
||||
echo "Update the module if it does not match before starting Frigate."
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euxo pipefail
|
||||
|
||||
# DEEPX DX-RT runtime (dxrtd daemon + the dx_engine python bindings) used by
|
||||
# the `deepx` detector plugin.
|
||||
#
|
||||
# A DEEPX install has three separately versioned pieces that have to agree:
|
||||
#
|
||||
# DX-RT runtime v3.4.2 this file, installed into the Frigate image
|
||||
# Kernel driver v2.6.0 the host, docker/deepx/user_installation.sh
|
||||
# NPU firmware v2.7.4 the module itself, flashed from the host
|
||||
#
|
||||
# DX-RT v3.4.2 requires driver 2.5.0 and firmware 2.7.0 as a minimum, so the
|
||||
# versions above sit just over that floor. The runtime is pinned rather than
|
||||
# resolved to "latest" so image builds stay reproducible, and so there is a
|
||||
# known version for the host side to match. A mismatch usually presents as
|
||||
# inference requests that are accepted but never complete, not as a startup
|
||||
# error, so bump all three together and keep the table in
|
||||
# docs/docs/frigate/installation.md in sync.
|
||||
dxrt_version="v3.4.2"
|
||||
|
||||
# SHA-256 of the .deb pinned above, one per architecture Frigate builds for.
|
||||
# DEEPX doesn't publish checksums or sign this release, so these are computed
|
||||
# once from a vetted download and hardcoded here as the actual integrity
|
||||
# check: a download that doesn't match one of these hashes is rejected before
|
||||
# it reaches apt-get, regardless of whether the mismatch came from the v3.4.2
|
||||
# tag being retargeted or the file being replaced upstream. Recompute both
|
||||
# (`sha256sum libdxrt-bin_*.deb`) when bumping dxrt_version.
|
||||
declare -A dxrt_sha256=(
|
||||
[amd64]="da782deac10acb1c997b52ea6d0160c06f112a68ad5e25035cfc8e16de0a8bdc"
|
||||
[arm64]="161223058938c54efadc581b33499210194da58e55cd5ade332e692ca0d649c2"
|
||||
)
|
||||
|
||||
# The release directory and the package filename both use the bare version,
|
||||
# while the git tag carries a "v".
|
||||
dxrt_release="${dxrt_version#v}"
|
||||
deb_arch=$(dpkg --print-architecture)
|
||||
deb_file="/tmp/libdxrt-bin_${dxrt_release}_${deb_arch}.deb"
|
||||
|
||||
# Fetch just the one package rather than the source archive: the repository
|
||||
# keeps every past release under release/, so the tag archive is several
|
||||
# hundred MB of .deb files we would immediately throw away. Note that
|
||||
# release/latest is a symlink and would not survive the archive anyway.
|
||||
wget -qO "${deb_file}" \
|
||||
"https://raw.githubusercontent.com/DEEPX-AI/dx_rt/${dxrt_version}/release/${dxrt_release}/libdxrt-bin_${dxrt_release}_${deb_arch}.deb"
|
||||
|
||||
expected_sha256="${dxrt_sha256[${deb_arch}]:-}"
|
||||
if [[ -z "${expected_sha256}" ]]; then
|
||||
echo "No pinned SHA-256 for architecture ${deb_arch}; refusing to install an unverified package."
|
||||
exit 1
|
||||
fi
|
||||
actual_sha256=$(sha256sum "${deb_file}" | cut -d' ' -f1)
|
||||
if [[ "${actual_sha256}" != "${expected_sha256}" ]]; then
|
||||
echo "SHA-256 mismatch for ${deb_file}:"
|
||||
echo " expected ${expected_sha256}"
|
||||
echo " actual ${actual_sha256}"
|
||||
echo "Refusing to install a package that doesn't match the pinned checksum."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Install with dpkg rather than apt. install_deps.sh finishes by purging gnupg
|
||||
# and clearing /var/lib/apt/lists, so apt here would have to re-fetch about
|
||||
# 10 MB of package indexes over the network before it could resolve anything,
|
||||
# which is a build failure waiting to happen for no benefit. The package only
|
||||
# depends on libc6, libgcc-s1, libncurses6, libstdc++6 and libtinfo6, all of
|
||||
# which the deps stage already has, and its postinst does nothing but run
|
||||
# ldconfig and print where the files went. dpkg fails loudly if a dependency
|
||||
# ever goes missing.
|
||||
dpkg -i "${deb_file}"
|
||||
|
||||
# The package bundles a dx_engine wheel per cpython ABI tag. Ask dpkg what it
|
||||
# installed rather than hardcoding a path, so this keeps working if DEEPX moves
|
||||
# the wheels between releases.
|
||||
package=$(dpkg-deb -f "${deb_file}" Package)
|
||||
py_tag="cp$(python3 -c 'import sys; print(f"{sys.version_info[0]}{sys.version_info[1]}")')"
|
||||
wheel=$(dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" | grep -- "${py_tag}" | sort | head -1 || true)
|
||||
|
||||
if [[ -z "${wheel}" ]]; then
|
||||
echo "No dx_engine wheel for ${py_tag} in ${package}. Wheels available:"
|
||||
dpkg -L "${package}" | grep -E "/dx_engine-.*\.whl$" || echo " (none)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pip3 install "${wheel}"
|
||||
|
||||
rm -f "${deb_file}"
|
||||
@ -1 +0,0 @@
|
||||
dxrtd
|
||||
@ -1 +0,0 @@
|
||||
dxrtd-pipeline
|
||||
@ -1,4 +0,0 @@
|
||||
#!/command/with-contenv bash
|
||||
# shellcheck shell=bash
|
||||
|
||||
exec logutil-service /dev/shm/logs/dxrtd
|
||||
@ -1 +0,0 @@
|
||||
longrun
|
||||
@ -1,43 +0,0 @@
|
||||
#!/command/with-contenv bash
|
||||
# shellcheck shell=bash
|
||||
# Take down the S6 supervision tree when the service fails
|
||||
|
||||
set -o errexit -o nounset -o pipefail
|
||||
|
||||
# Logs should be sent to stdout so that s6 can collect them
|
||||
|
||||
declare exit_code_container
|
||||
exit_code_container=$(cat /run/s6-linux-init-container-results/exitcode)
|
||||
readonly exit_code_container
|
||||
readonly exit_code_service="${1}"
|
||||
readonly exit_code_signal="${2}"
|
||||
readonly service="dxrtd"
|
||||
|
||||
echo "[INFO] Service ${service} exited with code ${exit_code_service} (by signal ${exit_code_signal})"
|
||||
|
||||
# dxrtd takes itself down when the machine has no DEEPX NPU, which is the
|
||||
# normal case: this service ships in the default image and most users do not
|
||||
# have the hardware. That self-shutdown reaches this script as exit 256 /
|
||||
# signal 15, indistinguishable from an operator stopping the container, so
|
||||
# without the marker written by run every machine lacking a DEEPX NPU would
|
||||
# have Frigate halted at startup.
|
||||
if [[ -f /dev/shm/.dxrtd-no-hardware ]]; then
|
||||
echo "[INFO] Service ${service} stopped because no DEEPX NPU is present; leaving the rest of Frigate running"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Past this point the service really was running, so only a genuine failure
|
||||
# should bring the container down.
|
||||
if [[ "${exit_code_service}" -eq 256 ]]; then
|
||||
if [[ "${exit_code_container}" -eq 0 ]]; then
|
||||
echo $((128 + exit_code_signal)) >/run/s6-linux-init-container-results/exitcode
|
||||
fi
|
||||
if [[ "${exit_code_signal}" -eq 15 ]]; then
|
||||
exec /run/s6/basedir/bin/halt
|
||||
fi
|
||||
elif [[ "${exit_code_service}" -ne 0 ]]; then
|
||||
if [[ "${exit_code_container}" -eq 0 ]]; then
|
||||
echo "${exit_code_service}" >/run/s6-linux-init-container-results/exitcode
|
||||
fi
|
||||
exec /run/s6/basedir/bin/halt
|
||||
fi
|
||||
@ -1 +0,0 @@
|
||||
dxrtd-log
|
||||
@ -1,41 +0,0 @@
|
||||
#!/command/with-contenv bash
|
||||
# shellcheck shell=bash
|
||||
# Start the DEEPX DX-RT runtime daemon (dxrtd)
|
||||
|
||||
set -o errexit -o nounset -o pipefail
|
||||
|
||||
# Logs should be sent to stdout so that s6 can collect them
|
||||
exec 2>&1
|
||||
|
||||
# The DEEPX kernel driver runs on the Docker host, not in this container, and
|
||||
# exposes the NPU as /dev/dxrt* device nodes that are passed through. Without
|
||||
# them there is no NPU to serve, so exit cleanly and let the rest of Frigate
|
||||
# start normally: this service ships in the default image and most users do
|
||||
# not have DEEPX hardware.
|
||||
if ! compgen -G "/dev/dxrt*" > /dev/null; then
|
||||
echo "[INFO] No DEEPX NPU device found, not starting dxrtd"
|
||||
# Tell finish that this shutdown is deliberate. Taking the service down
|
||||
# here arrives at finish as exit 256 / signal 15, which is exactly what an
|
||||
# operator stopping the container looks like, so the marker is the only
|
||||
# thing that tells the two apart.
|
||||
touch /dev/shm/.dxrtd-no-hardware
|
||||
exec s6-svc -Od .
|
||||
fi
|
||||
|
||||
# Reaching here means there is hardware to serve, so clear any marker left by
|
||||
# an earlier run before the device appeared.
|
||||
rm -f /dev/shm/.dxrtd-no-hardware
|
||||
|
||||
# dxrtd is a system-wide singleton. If the host's own dxrt.service is still
|
||||
# running it will refuse to start, so surface that up front rather than
|
||||
# leaving the user to decode the daemon's own error.
|
||||
if pgrep -x dxrtd > /dev/null; then
|
||||
echo "[ERROR] Another dxrtd instance is already running"
|
||||
echo "[ERROR] dxrtd may only run once per system. Stop the host service with:"
|
||||
echo "[ERROR] sudo systemctl disable --now dxrt.service"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[INFO] Starting dxrtd for DEEPX NPU device(s): $(echo /dev/dxrt*)"
|
||||
|
||||
exec /usr/local/bin/dxrtd
|
||||
@ -1 +0,0 @@
|
||||
30000
|
||||
@ -1 +0,0 @@
|
||||
longrun
|
||||
@ -37,6 +37,7 @@ device_globs=(
|
||||
"/dev/nvmap"
|
||||
"/dev/nvidia*"
|
||||
"/dev/memx*"
|
||||
"/dev/dxrt*"
|
||||
)
|
||||
|
||||
IFS=',' read -ra extra_globs <<< "${DEVICE_ACL_PATHS:-}"
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
|
||||
set -o errexit -o nounset -o pipefail
|
||||
|
||||
dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync /dev/shm/logs/dxrtd)
|
||||
dirs=(/dev/shm/logs/frigate /dev/shm/logs/go2rtc /dev/shm/logs/nginx /dev/shm/logs/certsync)
|
||||
|
||||
mkdir -p "${dirs[@]}"
|
||||
|
||||
|
||||
@ -1,20 +0,0 @@
|
||||
0 aeroplane
|
||||
1 bicycle
|
||||
2 bird
|
||||
3 boat
|
||||
4 bottle
|
||||
5 bus
|
||||
6 car
|
||||
7 cat
|
||||
8 chair
|
||||
9 cow
|
||||
10 diningtable
|
||||
11 dog
|
||||
12 horse
|
||||
13 motorbike
|
||||
14 person
|
||||
15 pottedplant
|
||||
16 sheep
|
||||
17 sofa
|
||||
18 train
|
||||
19 tvmonitor
|
||||
@ -985,6 +985,65 @@ memryx:
|
||||
# The .zip file must contain:
|
||||
# ├── ssdlite_mobilenet.dfp (a file ending with .dfp)
|
||||
# └── ssdlite_mobilenet_post.onnx (optional; only if the model includes a cropped post-processing network)
|
||||
deepx:
|
||||
title: DEEPX NPU
|
||||
models:
|
||||
- key: yolo
|
||||
label: YOLO (anchor / anchor-free / NMS-in-head)
|
||||
recommended: true
|
||||
download: No model is bundled with Frigate. Download a pre-compiled `.dxnn` model from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) or compile your own with DX-COM, then bind-mount it into the container and point the model's `path` at it.
|
||||
ui: |-
|
||||
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
|
||||
|
||||
| Field | Value |
|
||||
| ---------------------------------------- | -------------------------------------- |
|
||||
| **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` |
|
||||
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||
| **Object detection model input width** | `640` |
|
||||
| **Object detection model input height** | `640` |
|
||||
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
|
||||
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
|
||||
| **Model Input D Type** | `int` (Frigate's default value) |
|
||||
| **Object Detection Model Type** | `yolo-generic` |
|
||||
|
||||
The width and height must match the resolution the `.dxnn` file was compiled for. Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden.
|
||||
yaml: |-
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/model.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: yolo-generic
|
||||
width: 640
|
||||
height: 640
|
||||
- key: damoyolo
|
||||
label: DAMO-YOLO (TinyNAS-L20T / S / M / L)
|
||||
recommended: false
|
||||
download: No model is bundled with Frigate. Download a pre-compiled DAMO-YOLO `.dxnn` model (TinyNAS-L20T, S, M, or L size) from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), then bind-mount it into the container and point the model's `path` at it. All four sizes share one detection head, so nothing else has to be told apart.
|
||||
ui: |-
|
||||
Navigate to **Settings > System > Detection models** and select **DEEPX NPU** from the **Hardware** dropdown. Then, on the same model, open the **Custom Model** tab and configure:
|
||||
|
||||
| Field | Value |
|
||||
| ---------------------------------------- | -------------------------------------- |
|
||||
| **Custom object detector model path** | `/config/model_cache/deepx/model.dxnn` |
|
||||
| **Label map for custom object detector** | `/labelmap/coco-80.txt` |
|
||||
| **Object detection model input width** | `640` |
|
||||
| **Object detection model input height** | `640` |
|
||||
| **Model Input Pixel Color Format** | `rgb` (Frigate's default value) |
|
||||
| **Model Input Tensor Shape** | `nhwc` (Frigate's default value) |
|
||||
| **Model Input D Type** | `int` (Frigate's default value) |
|
||||
| **Object Detection Model Type** | `damo-yolo` |
|
||||
|
||||
The width and height must match the resolution the `.dxnn` file was compiled for. Quantization is baked into the compiled model, so no normalization is applied on the host and the input defaults do not need to be overridden.
|
||||
yaml: |-
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/model.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: damo-yolo
|
||||
width: 640
|
||||
height: 640
|
||||
tensorrt:
|
||||
title: TensorRT
|
||||
models:
|
||||
|
||||
@ -641,9 +641,15 @@ For detailed instructions on compiling models, refer to the [MemryX Compiler](ht
|
||||
|
||||
This detector is available for use with the DEEPX NPU, both the DX-M1 M.2 module and the DX-M1M on the Sixfab AI HAT+ for the Raspberry Pi 5. The configuration below applies unchanged to either form factor. DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
|
||||
|
||||
See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and passing the NPU through to the container.
|
||||
See the [installation docs](../frigate/installation.md#deepx-npu) for information on installing the DEEPX kernel driver and runtime on the host and passing the NPU through to the container.
|
||||
|
||||
To configure a DEEPX detector, set the `type` attribute to `deepx` and follow the configuration guide below.
|
||||
To run a model on a DEEPX NPU, list a `deepx` device on that model.
|
||||
|
||||
:::info
|
||||
|
||||
The DX-RT Python bindings are not part of the Frigate image. They are downloaded and installed into `/config/.local` the first time a DEEPX device is configured, verified against pinned checksums, and updated automatically when a Frigate release pins a new version. If the container has no internet access, see [Detector runtimes](/frigate/network_requirements#detector-runtimes) for how to provide the files yourself.
|
||||
|
||||
:::
|
||||
|
||||
### Configuration {#configuration-deepx}
|
||||
|
||||
@ -651,30 +657,43 @@ To configure a DEEPX detector, set the `type` attribute to `deepx` and follow th
|
||||
|
||||
Frigate does not bundle a model for this detector. Models must be compiled to DEEPX's `.dxnn` format.
|
||||
|
||||
The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of object detection models alongside a JSON file describing how each was compiled: YOLO variants (`model_type: yolo-generic`), SSD (`model_type: ssd`), and DAMO-YOLO (`model_type: damo-yolo`). Pick a model, download the `.dxnn`, bind-mount it into the container, and point `model.path` at it. Note which one you picked, since it determines the `model_type` and `model_format` values below.
|
||||
The quickest way to get one is the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), which publishes pre-compiled `.dxnn` files for a range of object detection models alongside a JSON file describing how each was compiled: YOLO variants (`model_type: yolo-generic`) and DAMO-YOLO (`model_type: damo-yolo`). Pick a model, download the `.dxnn`, bind-mount it into the container, and point the model's `path` at it. Note which one you picked, since it determines the `model_type` value below.
|
||||
|
||||
`model_type` is set on the detector rather than on the root `model` block. Frigate ignores a `model` block nested under a detector, and the root `model.model_type` defaults to `ssd`, so leaving it unset would silently decode a YOLO model with the SSD decoder and report no detections at all. The detector's own `model_type` defaults to `yolo-generic` and overrides the root value.
|
||||
```yaml
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/model.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: yolo-generic
|
||||
width: 640
|
||||
height: 640
|
||||
```
|
||||
|
||||
A DEEPX device is `PCIe:<index>`, as reported on the detector settings page. The NPU daemon multiplexes across processes, so the same device may be listed more than once to run additional inference processes against it:
|
||||
|
||||
```yaml
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
- deepx:PCIe:1
|
||||
```
|
||||
|
||||
Alternatively, compile your own model with the DX-COM compiler.
|
||||
|
||||
`model.width` and `model.height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden.
|
||||
`width` and `height` must match the resolution the model was compiled for. Quantization parameters are baked into the `.dxnn` file at compile time, so no normalization is applied on the host and Frigate's default `input_tensor`, `input_pixel_format`, and `input_dtype` values do not need to be overridden.
|
||||
|
||||
#### Label maps
|
||||
|
||||
YOLO and DAMO-YOLO models from the DEEPX ModelZoo are trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. SSD models are trained on Pascal VOC's 20-class label set instead, and need `labelmap_path` set to `/labelmap/voc-20.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type for any of these.
|
||||
Every supported model from the DEEPX ModelZoo is trained on the standard 80-class COCO label set, so `labelmap_path` must be set to `/labelmap/coco-80.txt`. Frigate's default label map uses an extended 91-class COCO scheme, and leaving it in place will cause detections to be reported as the wrong object type.
|
||||
|
||||
#### Decoder options {#deepx-decoder-options}
|
||||
|
||||
`model_format`, `ppu`, and `anchors` belong to the DEEPX detector rather than to a model. A `models` entry carries no per-detector options, so there is currently no way to set them and every DEEPX detector runs with their defaults, `model_format: auto` and `ppu: false`. In practice a `.dxnn` file therefore has to be one the `auto` inference can read: an anchor-free `yolo-generic` head, or DAMO-YOLO, which takes no layout at all. The sections below describe what each option selects, for when the config grows a place to set them.
|
||||
|
||||
#### YOLO model formats {#deepx-yolo-model-formats}
|
||||
|
||||
For `model_type: yolo-generic`, DX-COM preserves the detection head of the model it compiles, so the head's tensor layout determines how the output has to be read. Set `model_format` to the layout your `.dxnn` file was compiled with:
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
deepx:
|
||||
type: deepx
|
||||
model_format: anchor_free
|
||||
```
|
||||
|
||||
Supported values are three decode families:
|
||||
For `model_type: yolo-generic`, DX-COM preserves the detection head of the model it compiles, so the head's tensor layout determines how the output has to be read. `model_format` names the layout the `.dxnn` file was compiled with, across three decode families:
|
||||
|
||||
| `model_format` | Output layout | PPU support |
|
||||
| -------------- | -------------- | ----------- |
|
||||
@ -682,84 +701,45 @@ Supported values are three decode families:
|
||||
| `anchor_free` | Anchor-free, no objectness, boxes already in pixels | Yes |
|
||||
| `nms_in_head` | NMS runs in the detection head, output is final corner boxes | No |
|
||||
|
||||
Leaving `model_format` at its `auto` default falls back to inferring the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for anchor-free heads. Setting it explicitly is recommended.
|
||||
The `auto` default infers the layout from the output shape. That inference cannot distinguish an anchor-based head from an anchor-free one, so it is only reliable for anchor-free heads.
|
||||
|
||||
Two mistakes this option prevents:
|
||||
Two mistakes an explicit layout prevents:
|
||||
|
||||
- Decoding an anchor-free model with the anchor formula squares the box dimensions, producing boxes hundreds of thousands of pixels wide that clip to the entire frame.
|
||||
- Decoding an anchor-based model as anchor-free reads the objectness column as a class score, so confidences and class IDs are both wrong.
|
||||
|
||||
`nms_in_head` models run NMS inside the head, so `nms_threshold` does not apply to them.
|
||||
|
||||
#### SSD models
|
||||
|
||||
For `model_type: ssd`, no `model_format` is needed. DX-COM keeps SSD's postprocessing inside the compiled model: the `.dxnn` runs the NPU head into a small CPU subgraph that concatenates the per-feature-map outputs, applies the softmax, and decodes the prior-box regression all the way to corner-form boxes. Only NMS is left for the host, so nothing about the backbone has to be named. This applies to all three ModelZoo backbones (MobileNetV1, MobileNetV2-Lite, VGG16) even though they emit different numbers of boxes (3000 and 8732).
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
deepx:
|
||||
type: deepx
|
||||
model_type: ssd
|
||||
|
||||
model:
|
||||
width: 300
|
||||
height: 300
|
||||
labelmap_path: /labelmap/voc-20.txt
|
||||
```
|
||||
`nms_in_head` models run NMS inside the head, so the detector's `nms_threshold` does not apply to them.
|
||||
|
||||
#### DAMO-YOLO models
|
||||
|
||||
For `model_type: damo-yolo`, no `model_format` is needed — all four sizes (TinyNAS-L20T, S, M, L) share one detection head, differing only in backbone depth/width. Leave `model_format` at its `auto` default:
|
||||
For `model_type: damo-yolo` no layout is needed. All four sizes (TinyNAS-L20T, S, M, L) share one detection head, differing only in backbone depth/width, and the decoder reads that one layout directly:
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
deepx:
|
||||
type: deepx
|
||||
models:
|
||||
- devices:
|
||||
- deepx:PCIe:0
|
||||
path: /config/model_cache/deepx/model.dxnn
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
model_type: damo-yolo
|
||||
|
||||
model:
|
||||
width: 640
|
||||
height: 640
|
||||
labelmap_path: /labelmap/coco-80.txt
|
||||
width: 640
|
||||
height: 640
|
||||
```
|
||||
|
||||
#### PPU models
|
||||
|
||||
Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. PPU support is currently only available for `model_type: yolo-generic` — there is no confirmed PPU record layout for SSD or DAMO-YOLO yet, and setting `ppu: true` with either is rejected at startup.
|
||||
|
||||
Set `ppu: true` on the detector when using a PPU-compiled yolo-generic model:
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
deepx:
|
||||
type: deepx
|
||||
model_format: anchor_free
|
||||
ppu: true
|
||||
score_threshold: 0.25
|
||||
nms_threshold: 0.45
|
||||
```
|
||||
Models compiled with Post-Processing Unit (PPU) support perform candidate selection on the NPU itself and emit a list of surviving detections rather than raw feature maps. This leaves only the box decode and NMS for the host, which lowers CPU usage. PPU support is only available for `model_type: yolo-generic` — there is no confirmed PPU record layout for DAMO-YOLO yet, and combining the two is rejected at startup.
|
||||
|
||||
`ppu` and `model_format` are independent: `ppu` says whether candidate selection already happened on the NPU, and `model_format` says how to read the boxes that come back. Both must match how the model was compiled.
|
||||
|
||||
`model_format` is required whenever `ppu: true` is set. A PPU record has the same fixed-width layout for every yolo-generic variant, so unlike the raw output path there is no shape to infer the layout from — leaving it at `auto` is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch.
|
||||
`model_format` is required whenever `ppu` is set. A PPU record has the same fixed-width layout for every yolo-generic variant, so unlike the raw output path there is no shape to infer the layout from — leaving it at `auto` is rejected at startup rather than risking a silent anchor-free/anchor-based mismatch. Since neither option can currently be set, PPU-compiled models cannot be used.
|
||||
|
||||
DX-COM cannot compile `nms_in_head` models with PPU support, since their heads already produce final detections. Combining the two is rejected at startup.
|
||||
|
||||
#### Multiple NPUs
|
||||
##### Anchors {#deepx-ppu-anchors}
|
||||
|
||||
`device_ids` defaults to `auto`, which binds the detector to every NPU it can find. To pin a detector to specific devices, set it to a comma-separated list of the indices you want it to use. This allows multiple detectors to be configured against separate modules:
|
||||
`model_format: anchor` with `ppu` also needs `anchors`. A PPU record names its anchor by index and says nothing about the table behind that index, and anchor tables differ between models that share this output layout, so there is nothing safe to default to — the wrong table rescales every box by the wrong amount, and the boxes still look plausible enough that nothing reports an error. It is required rather than defaulted for the same reason `model_format` is.
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
deepx_0:
|
||||
type: deepx
|
||||
device_ids: "0"
|
||||
deepx_1:
|
||||
type: deepx
|
||||
device_ids: "1,2"
|
||||
```
|
||||
|
||||
A yaml list (`device_ids: [0, 1]`) is accepted as well, but the comma-separated string is what the config UI writes.
|
||||
The table is copied out of the config the model was trained with. It is one group of `width,height` pairs per feature-map layer, groups separated by `;`, in stride order 8, 16, 32, for example `10,13,16,30,33,23;30,61,62,45,59,119;116,90,156,198,373,326`. The nested list form a model config usually declares them in is accepted too. Every group must hold the same number of pairs, since a record's anchor index is a single number that has to mean the same thing on every layer. `anchors` is rejected on any other combination, because no other decode path reads it: `anchor_free` and `nms_in_head` models carry their box geometry in the output, and the non-PPU `anchor` path receives boxes already in pixels.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@ -66,7 +66,7 @@ Frigate supports multiple different detectors that work on different types of ha
|
||||
- Runs best with tiny, small, or medium-size models
|
||||
|
||||
- <CommunityBadge /> [DEEPX](#deepx-npu): The DEEPX NPU is available in m.2 format and as a HAT+ for the Raspberry Pi 5, allowing for a wide range of compatibility with devices.
|
||||
- [Supports YOLO, SSD, and DAMO-YOLO model architectures](../../configuration/object_detectors#deepx-npu)
|
||||
- [Supports YOLO and DAMO-YOLO model architectures](../../configuration/object_detectors#deepx-npu)
|
||||
- Runs best with tiny or small size models
|
||||
- Runs efficiently on low power hardware
|
||||
|
||||
@ -267,17 +267,34 @@ Frigate supports the DEEPX NPU in both of its form factors: the **DX-M1** M.2 mo
|
||||
|
||||
DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://sixfab.com).
|
||||
|
||||
The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO, SSD, and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and YOLO models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage.
|
||||
The NPU runs models compiled to the `.dxnn` format with DEEPX's DX-COM compiler. Pre-compiled YOLO and DAMO-YOLO models can be downloaded from the [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo), and YOLO models compiled with Post-Processing Unit (PPU) support move candidate selection onto the NPU, which reduces host CPU usage.
|
||||
|
||||
The DEEPX kernel driver has to be installed on the Docker host before the NPU can be used. See the [installation docs](installation.md#deepx-npu) for the setup steps.
|
||||
The DEEPX kernel driver, the DX-RT runtime, and the `dxrtd` daemon all run on the Docker host rather than inside the Frigate container, and have to be installed there before the NPU can be used. Frigate connects to the daemon over its socket, so the NPU stays available to other programs on the host at the same time. See the [installation docs](installation.md#deepx-npu) for the setup steps.
|
||||
|
||||
Detailed information is available [in the detector docs](/configuration/object_detectors#deepx-npu).
|
||||
|
||||
| Name | Input Size | Inference Time |
|
||||
| ------- | ---------- | -------------- |
|
||||
| YOLOv8s | 640 | ~ 19 ms |
|
||||
The [DEEPX ModelZoo](https://developer.deepx.ai/modelzoo) publishes pre-compiled `.dxnn` files for many YOLO variants as well as the models below:
|
||||
|
||||
Inference times were measured on a Raspberry Pi 5. The host platform affects the pre- and post-processing cost rather than the inference itself.
|
||||
| Model | Input Size | DX-M1 Inference Time |
|
||||
| ---------------------- | ---------- | -------------------- |
|
||||
| DAMO-YOLO-T | 640 | ~ 5.0 ms |
|
||||
| DAMO-YOLO TinyNAS-L20T | 640 | ~ 5.1 ms |
|
||||
| DAMO-YOLO-S | 640 | ~ 6.0 ms |
|
||||
| DAMO-YOLO TinyNAS-L25S | 640 | ~ 6.0 ms |
|
||||
| DAMO-YOLO-M | 640 | ~ 6.8 ms |
|
||||
| DAMO-YOLO TinyNAS-L20M | 640 | ~ 6.8 ms |
|
||||
| DAMO-YOLO-L | 640 | ~ 9.0 ms |
|
||||
| SSD (MobileNetV1) | 300 | ~ 0.5 ms |
|
||||
| SSD (MobileNetV2-Lite) | 300 | ~ 0.6 ms |
|
||||
| SSD (VGG-16) | 300 | ~ 3.3 ms |
|
||||
|
||||
These are DEEPX's own figures for the DX-M1, derived from the frames per second published in the ModelZoo. They cover the NPU alone and exclude the pre- and post-processing Frigate does on the host, so the inference speed Frigate reports will be higher, and noticeably so on a slower host such as a Raspberry Pi 5.
|
||||
|
||||
:::note
|
||||
|
||||
The SSD rows describe what the NPU can run, not what Frigate can use. All three SSD models are trained on Pascal VOC rather than COCO, so their labels do not match the object vocabulary used by `objects.track` and the rest of Frigate. The `deepx` detector has no SSD decoder and rejects `model_type: ssd` at startup. Use a DAMO-YOLO or YOLO model instead.
|
||||
|
||||
:::
|
||||
|
||||
### Nvidia Jetson
|
||||
|
||||
|
||||
@ -394,37 +394,26 @@ DEEPX NPU support in Frigate is developed and maintained by [Sixfab](https://six
|
||||
|
||||
#### Versions
|
||||
|
||||
A DEEPX install has three separately versioned pieces, and all three have to agree. The DX-RT runtime is built into the Frigate image, so the two host-side pieces must be brought up to match it:
|
||||
A DEEPX install has several separately versioned pieces, and they all have to agree. The driver, the runtime, and the daemon live on the Docker host; Frigate itself carries only the Python bindings, which it downloads on first start:
|
||||
|
||||
| Component | Version | Installed on | Installed by |
|
||||
| ------------- | -------- | ------------- | -------------------------- |
|
||||
| DX-RT runtime | `v3.4.2` | Frigate image | Built in, no action needed |
|
||||
| Kernel driver | `v2.6.0` | Docker host | `user_installation.sh` |
|
||||
| NPU firmware | `v2.7.4` | The module | Flashed from the host |
|
||||
| Component | Version | Installed on | Installed by |
|
||||
| -------------- | -------- | ------------ | ------------------------- |
|
||||
| Kernel driver | `v2.6.0` | Host | `user_installation.sh` |
|
||||
| DX-RT runtime | `v3.4.0` | Host | `user_installation.sh` |
|
||||
| NPU firmware | `v2.7.4` | The module | Flashed from the host |
|
||||
| DX-RT bindings | `v3.4.0` | Frigate | Downloaded at first start |
|
||||
|
||||
:::warning
|
||||
|
||||
A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check all three versions before anything else.
|
||||
A version mismatch does not produce a startup error. It typically shows up as inference requests that are accepted but never return a result, so detections simply stop appearing while Frigate looks healthy. If that happens after a Frigate upgrade, check every version in the table before anything else.
|
||||
|
||||
:::
|
||||
|
||||
The firmware version cannot be read without the DX-RT runtime installed on the host, so checking or updating it is a manual step done before Frigate is running:
|
||||
|
||||
1. Install the DX-RT runtime on the host by following DEEPX's own instructions. If you are using Sixfab hardware, their [AI HAT+ quickstart](https://docs.sixfab.com/docs/ai-hat-plus-raspberry-pi-5-quickstart) covers this.
|
||||
2. Check the reported firmware version and update the module if it does not match the table above.
|
||||
3. **Remove or disable the host runtime before starting the Frigate container.**
|
||||
|
||||
:::danger
|
||||
|
||||
Step 3 is not optional when running Frigate in Docker. The DX-RT daemon (`dxrtd`) may only run once per system, and Frigate runs its own copy inside the container. Leaving the host runtime active means two daemons competing for the same NPU, and the container's copy will fail to start.
|
||||
|
||||
This only applies to Docker. If you run Frigate directly on the host, it uses the host's runtime and there is no conflict.
|
||||
|
||||
:::
|
||||
The installation script installs the DX-RT runtime on the host and enables `dxrt.service`, so the daemon starts at boot and any other program on the host can share the NPU with Frigate. Check the firmware version with `dxrt-cli --status` and update the module if it does not match the table above.
|
||||
|
||||
#### Installation
|
||||
|
||||
The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate.
|
||||
The DEEPX kernel driver must be installed on the host rather than in the container, because containers share the host kernel and cannot load kernel modules. Installing it creates the `/dev/dxrt*` device nodes that are passed through to Frigate. The same script installs the DX-RT runtime and enables `dxrt.service`, the daemon that owns the NPU and hands work to it on behalf of Frigate and anything else on the host.
|
||||
|
||||
1. Copy or download [this script](https://github.com/blakeblackshear/frigate/blob/dev/docker/deepx/user_installation.sh).
|
||||
2. Ensure it has execution permissions with `sudo chmod +x user_installation.sh`
|
||||
@ -437,28 +426,46 @@ Confirm the NPU is visible before continuing:
|
||||
ls /dev/dxrt*
|
||||
```
|
||||
|
||||
:::note
|
||||
|
||||
The installation script disables the host's `dxrt.service` if it finds one, for the reason described above. If you install or reinstall the DEEPX runtime on the host afterwards, disable it again before starting the container:
|
||||
Then confirm the daemon is running:
|
||||
|
||||
```bash
|
||||
sudo systemctl disable --now dxrt.service
|
||||
systemctl is-active dxrt.service
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
#### Setup
|
||||
|
||||
To set up Frigate, follow the default installation instructions, for example: `ghcr.io/blakeblackshear/frigate:stable`
|
||||
|
||||
Next, grant Docker access to the NPU by adding the device to your `docker-compose.yml` file:
|
||||
#### Docker configuration
|
||||
|
||||
Frigate needs the NPU device node and the daemon's socket:
|
||||
|
||||
```yaml
|
||||
devices:
|
||||
- /dev/dxrt0
|
||||
services:
|
||||
frigate:
|
||||
devices:
|
||||
- /dev/dxrt0:/dev/dxrt0
|
||||
volumes:
|
||||
- /tmp/dxrt_dynamic_ipc.sock:/tmp/dxrt_dynamic_ipc.sock
|
||||
```
|
||||
|
||||
If you can't use Docker Compose, add `--device /dev/dxrt0` to your `docker run` command.
|
||||
If you can't use Docker Compose, add `--device /dev/dxrt0:/dev/dxrt0 -v /tmp/dxrt_dynamic_ipc.sock:/tmp/dxrt_dynamic_ipc.sock` to your `docker run` command.
|
||||
|
||||
Add one `--device` per NPU, contiguously from `/dev/dxrt0`, since the client stops enumerating at the first gap.
|
||||
|
||||
:::note
|
||||
|
||||
Restart `dxrt.service` before starting the container, not after. A bind-mounted socket pins the inode it saw at container start, so recreating the socket afterwards leaves the container holding a deleted one.
|
||||
|
||||
:::
|
||||
|
||||
The device node is needed as well as the socket, because the client opens the NPU directly even though the daemon arbitrates access. Without it, inference fails with `Device not found`.
|
||||
|
||||
`/dev/shm` does not need sharing. The daemon publishes a shared memory segment for device monitoring, which `dxtop` and device status read, but inference does not.
|
||||
|
||||
`dxrtd` listens on both an abstract socket and `/tmp/dxrt_dynamic_ipc.sock`. Only the second crosses into a container, so Frigate names it through `DXRT_DYNAMIC_IPC_ENDPOINT` on your behalf. Set that variable yourself only if you moved the socket, in which case set it for the daemon too, through a systemd drop-in.
|
||||
|
||||
The DX-RT python bindings are not shipped in the Frigate image. Frigate downloads them on first start when a DEEPX detector is configured, and caches them under `/config`.
|
||||
|
||||
#### Configuration
|
||||
|
||||
|
||||
@ -273,6 +273,16 @@ def detect_memryx() -> DetectionHardware | None:
|
||||
return _hardware("memryx", "memryx", "MemryX MX3", units)
|
||||
|
||||
|
||||
def detect_deepx() -> DetectionHardware | None:
|
||||
"""Find DEEPX NPUs by their device nodes."""
|
||||
units = _dev_units("dxrt*", "deepx:PCIe:{index}", "PCIe")
|
||||
|
||||
if not units:
|
||||
return None
|
||||
|
||||
return _hardware("deepx", "deepx", "DEEPX NPU", units)
|
||||
|
||||
|
||||
def detect_rockchip() -> DetectionHardware | None:
|
||||
"""Find a Rockchip NPU by reading the SoC from the device tree."""
|
||||
compatible = _read(f"{PROC_ROOT}/device-tree/compatible")
|
||||
@ -319,6 +329,7 @@ PROBES = (
|
||||
detect_coral_usb,
|
||||
detect_hailo,
|
||||
detect_memryx,
|
||||
detect_deepx,
|
||||
detect_intel_npu,
|
||||
detect_intel_gpu,
|
||||
detect_nvidia_gpu,
|
||||
|
||||
@ -1,8 +1,10 @@
|
||||
"""DEEPX NPU detector running compiled .dxnn models via the DX-RT runtime."""
|
||||
|
||||
import atexit
|
||||
import glob
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
from enum import Enum
|
||||
from typing import Literal
|
||||
|
||||
@ -13,49 +15,144 @@ from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
from frigate.detectors.detection_api import DetectionApi
|
||||
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
|
||||
from frigate.util.model import post_process_yolo
|
||||
from frigate.util.runtime_deps import Artifact, ArtifactKind, RuntimeManifest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTOR_KEY = "deepx"
|
||||
|
||||
DXNN_DEVICES_ENV = "DXNN_DEVICES"
|
||||
# Installed at first start rather than shipped in the image. DEEPX's PyPI
|
||||
# wheels are byte-identical to the ones libdxrt-bin carries, so the host
|
||||
# runtime and the container bindings come from one vendor build.
|
||||
DXRT_VERSION = "3.4.0"
|
||||
|
||||
# Stands in for "let Frigate pick the devices". A plain empty string would be
|
||||
# written back to the config as a deletion, so the no-op needs a real value.
|
||||
DEVICE_IDS_AUTO = "auto"
|
||||
# Where dxrtd, running on the host, accepts client connections. The abstract
|
||||
# socket it also listens on cannot be reached from a container (verified).
|
||||
DXRT_IPC_ENDPOINT_ENV = "DXRT_DYNAMIC_IPC_ENDPOINT"
|
||||
DXRT_IPC_SOCKET = "/tmp/dxrt_dynamic_ipc.sock"
|
||||
|
||||
# DX-RT decides whether dxrtd runs by scanning /proc for a cmdline holding
|
||||
# "dxrtd" (checkService, lib/device_pool/service_util.cpp). The daemon lives on
|
||||
# the host, and sharing its PID namespace is closed to us because s6-overlay
|
||||
# demands PID 1, so a placeholder carrying that name answers the scan while the
|
||||
# real work goes to the host over DXRT_IPC_SOCKET.
|
||||
_SERVICE_PLACEHOLDER: subprocess.Popen | None = None
|
||||
|
||||
|
||||
def service_is_visible() -> bool:
|
||||
"""Whether a process named dxrtd is visible in this PID namespace."""
|
||||
for path in glob.glob("/proc/[0-9]*/cmdline"):
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
if b"dxrtd" in f.read():
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def satisfy_service_check(socket_path: str) -> None:
|
||||
"""Make DX-RT's liveness scan succeed when the daemon runs on the host.
|
||||
|
||||
A no-op when a dxrtd is visible, or when `socket_path` is absent so a
|
||||
genuinely stopped daemon still reports itself.
|
||||
"""
|
||||
global _SERVICE_PLACEHOLDER
|
||||
|
||||
if _SERVICE_PLACEHOLDER is not None and _SERVICE_PLACEHOLDER.poll() is None:
|
||||
return
|
||||
|
||||
if service_is_visible():
|
||||
return
|
||||
|
||||
if not os.path.exists(socket_path):
|
||||
logger.warning(
|
||||
"No dxrtd socket at %s. Start dxrt.service on the host and mount "
|
||||
"the socket into the container.",
|
||||
socket_path,
|
||||
)
|
||||
return
|
||||
|
||||
try:
|
||||
_SERVICE_PLACEHOLDER = subprocess.Popen(
|
||||
["dxrtd", "infinity"],
|
||||
executable="/bin/sleep",
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
except OSError as err:
|
||||
logger.error("Could not satisfy the DX-RT service check: %s", err)
|
||||
return
|
||||
|
||||
atexit.register(_stop_service_placeholder)
|
||||
logger.debug(
|
||||
"Standing in for the host dxrtd in this namespace as pid %d",
|
||||
_SERVICE_PLACEHOLDER.pid,
|
||||
)
|
||||
|
||||
|
||||
def _stop_service_placeholder() -> None:
|
||||
if _SERVICE_PLACEHOLDER is not None and _SERVICE_PLACEHOLDER.poll() is None:
|
||||
_SERVICE_PLACEHOLDER.terminate()
|
||||
|
||||
|
||||
# PyPI paths carry a per-file digest, so each URL is written out in full; a
|
||||
# version bump has to replace it, which the manifest tests check for.
|
||||
DEEPX_MANIFEST = RuntimeManifest(
|
||||
name=DETECTOR_KEY,
|
||||
version=DXRT_VERSION,
|
||||
artifacts=(
|
||||
Artifact(
|
||||
url=(
|
||||
"https://files.pythonhosted.org/packages/de/45/"
|
||||
"1ca593e1c4ed868618658e07adeffc1fb44b92655d1ba48a6c76942616ee/"
|
||||
"dx_engine-3.4.0-cp311-cp311-manylinux_2_27_aarch64.whl"
|
||||
),
|
||||
sha256="ec4188e0a598a164bc04312482f70159be1a3d888284dcf52746054138101ec7",
|
||||
kind=ArtifactKind.wheel,
|
||||
machines=("aarch64",),
|
||||
),
|
||||
Artifact(
|
||||
url=(
|
||||
"https://files.pythonhosted.org/packages/b0/32/"
|
||||
"d52b33d5b85f7d8565e3e2d153f19db6d506cd5504339f31c6553eae06c6/"
|
||||
"dx_engine-3.4.0-cp311-cp311-manylinux_2_27_x86_64.whl"
|
||||
),
|
||||
sha256="161fde8428fc8aea95c560d878bc68e8d1c657c18a0eb04840de4bd212f3477f",
|
||||
kind=ArtifactKind.wheel,
|
||||
machines=("x86_64",),
|
||||
),
|
||||
),
|
||||
import_check="dx_engine",
|
||||
)
|
||||
|
||||
|
||||
class DeepxModelTypeEnum(str, Enum):
|
||||
"""The subset of Frigate's model types the DEEPX decoders can read.
|
||||
|
||||
Frigate keeps `model_type` on the root `model` block and ignores a nested
|
||||
one, so it is not selectable per detector in the config UI, and an
|
||||
omitted value silently defaults to `ssd`. Repeating the supported values
|
||||
here as a detector option makes the choice visible in the form; the
|
||||
detector writes it back onto the merged model config at startup so the
|
||||
rest of Frigate agrees with the decoder.
|
||||
Frigate reads `model_type` off the root `model` block, where an omitted
|
||||
value defaults to `ssd`; listing the supported ones here makes the choice
|
||||
visible, and the detector writes it back at startup. `ssd` is left out on
|
||||
purpose: the ModelZoo's SSD models carry Pascal VOC's 20 class names, so it
|
||||
would mean an empty label map, not a config error.
|
||||
"""
|
||||
|
||||
# PPU-compiled models still use yolo-generic; the `ppu` option is what
|
||||
# selects the decode path, since there is no PPU-specific model type.
|
||||
yologeneric = ModelTypeEnum.yologeneric.value
|
||||
ssd = ModelTypeEnum.ssd.value
|
||||
damoyolo = ModelTypeEnum.damoyolo.value
|
||||
|
||||
|
||||
class ModelFormatEnum(str, Enum):
|
||||
"""How a .dxnn file's output is decoded.
|
||||
|
||||
DX-COM preserves the detection head of the source model, so the exact
|
||||
layout has to be told apart to read it. Values are grouped by which
|
||||
`model_type` they apply to; a model_validator on DeepxDetectorConfig
|
||||
enforces that a given value is only used with its matching type.
|
||||
"""How a .dxnn file's output is decoded. DX-COM preserves the source
|
||||
model's head, so the layout has to be named; a model_validator keeps each
|
||||
value with its own `model_type`.
|
||||
"""
|
||||
|
||||
# "no explicit layout", and the default. model_type=yolo-generic infers
|
||||
# the layout from the output shape; model_type=damo-yolo takes no layout
|
||||
# at all. This is a real enum member rather than None so the config form
|
||||
# always round-trips a value -- a null is written back as a deletion.
|
||||
# The default. yolo-generic infers the layout from the output shape and
|
||||
# damo-yolo takes none. A real member rather than None so the config form
|
||||
# round-trips a value -- a null is written back as a deletion.
|
||||
auto = "auto"
|
||||
# model_type=yolo-generic: which detection-head tensor layout the .dxnn
|
||||
# was compiled with.
|
||||
@ -89,13 +186,9 @@ NMS_IN_HEAD_FORMATS = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
# Anchor table for anchor-based PPU models, keyed by stride.
|
||||
PPU_ANCHORS = {
|
||||
8: np.array([[10, 13], [16, 30], [33, 23]], dtype=np.float32),
|
||||
16: np.array([[30, 61], [62, 45], [59, 119]], dtype=np.float32),
|
||||
32: np.array([[116, 90], [156, 198], [373, 326]], dtype=np.float32),
|
||||
}
|
||||
# The feature-map strides a PPU record's layer index selects between.
|
||||
PPU_STRIDES = np.array([8, 16, 32], dtype=np.float32)
|
||||
PPU_ANCHOR_LAYERS = len(PPU_STRIDES)
|
||||
|
||||
# Byte layout of the fixed-width detection record the PPU emits.
|
||||
PPU_RECORD_SIZE = 32
|
||||
@ -110,72 +203,89 @@ PPU_LABEL_BYTES = (24, 28)
|
||||
DAMOYOLO_STRIDES = (8, 16, 32)
|
||||
|
||||
|
||||
def detect_device_count() -> int:
|
||||
"""Count the NPU devices visible to this process.
|
||||
|
||||
Falls back from the DX-RT API to a scan of the passed-through device
|
||||
nodes, then to a single device.
|
||||
"""
|
||||
try:
|
||||
from dx_engine import get_device_count
|
||||
|
||||
count = get_device_count()
|
||||
except (ImportError, RuntimeError) as err:
|
||||
logger.debug("Could not query DX-RT for a device count: %s", err)
|
||||
else:
|
||||
if count > 0:
|
||||
return count
|
||||
|
||||
devices = glob.glob("/dev/dxrt*")
|
||||
|
||||
return len(devices) if devices else 1
|
||||
|
||||
|
||||
def parse_device_ids(value: str) -> list[int]:
|
||||
"""Parse a comma-separated list of NPU device indices.
|
||||
|
||||
The `auto` sentinel and an empty value both mean "nothing was picked",
|
||||
which leaves device selection to `resolve_devices`.
|
||||
"""
|
||||
if value.strip().lower() == DEVICE_IDS_AUTO:
|
||||
return []
|
||||
|
||||
return [int(part) for part in value.split(",") if part.strip()]
|
||||
|
||||
|
||||
def resolve_devices(configured: str) -> list[int]:
|
||||
"""Resolve which NPU device indices this detector should bind to."""
|
||||
if configured:
|
||||
# already validated by DeepxDetectorConfig
|
||||
devices = parse_device_ids(configured)
|
||||
|
||||
if devices:
|
||||
return devices
|
||||
|
||||
env_value = os.environ.get(DXNN_DEVICES_ENV)
|
||||
|
||||
if env_value:
|
||||
try:
|
||||
devices = parse_device_ids(env_value)
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
"Ignoring unparsable %s=%s, detecting devices instead",
|
||||
DXNN_DEVICES_ENV,
|
||||
env_value,
|
||||
)
|
||||
def _flatten(group):
|
||||
"""Yield a nested sequence one level flat, leaving a flat one alone."""
|
||||
for item in group:
|
||||
if isinstance(item, (list, tuple)):
|
||||
yield from item
|
||||
else:
|
||||
if devices:
|
||||
return devices
|
||||
yield item
|
||||
|
||||
return list(range(detect_device_count()))
|
||||
|
||||
def parse_anchors(value: str) -> dict[int, np.ndarray]:
|
||||
"""Parse the anchor table an anchor-based PPU model was trained with.
|
||||
|
||||
One group per layer separated by ";" in PPU_STRIDES order, each a flat run
|
||||
of width,height pairs, keyed by stride the way decode_ppu_anchor reads it.
|
||||
"""
|
||||
groups = [group for group in value.split(";") if group.strip()]
|
||||
|
||||
if len(groups) != PPU_ANCHOR_LAYERS:
|
||||
strides = ", ".join(str(int(stride)) for stride in PPU_STRIDES)
|
||||
raise ValueError(
|
||||
f"expected {PPU_ANCHOR_LAYERS} anchor groups separated by ';', one "
|
||||
f"per feature-map layer for strides {strides}, got {len(groups)}"
|
||||
)
|
||||
|
||||
parsed = []
|
||||
|
||||
for group in groups:
|
||||
try:
|
||||
numbers = [float(part) for part in group.split(",") if part.strip()]
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f'anchor group "{group.strip()}" is not a comma-separated list '
|
||||
"of numbers"
|
||||
) from None
|
||||
|
||||
if not numbers or len(numbers) % 2:
|
||||
raise ValueError(
|
||||
f'anchor group "{group.strip()}" is not an even-length list of '
|
||||
"width,height pairs"
|
||||
)
|
||||
|
||||
if any(number <= 0 for number in numbers):
|
||||
raise ValueError(
|
||||
f'anchor group "{group.strip()}" has a width or height that is '
|
||||
"not positive"
|
||||
)
|
||||
|
||||
parsed.append(np.array(numbers, dtype=np.float32).reshape(-1, 2))
|
||||
|
||||
# a record picks its anchor with a bare index and its layer separately, so
|
||||
# that index only means one thing if every layer offers the same count
|
||||
if len({len(anchors) for anchors in parsed}) != 1:
|
||||
counts = ", ".join(str(len(anchors)) for anchors in parsed)
|
||||
raise ValueError(
|
||||
f"every anchor group must hold the same number of pairs, got {counts}"
|
||||
)
|
||||
|
||||
return {
|
||||
int(stride): anchors
|
||||
for stride, anchors in zip(PPU_STRIDES, parsed, strict=True)
|
||||
}
|
||||
|
||||
|
||||
def resolve_device(configured: str) -> int:
|
||||
"""Resolve the NPU index from the device half of a `deepx:...` string:
|
||||
empty, a bare index, or `PCIe:<index>` as the hardware probe writes it.
|
||||
"""
|
||||
if not configured:
|
||||
return 0
|
||||
|
||||
index = configured.rsplit(":", 1)[-1]
|
||||
|
||||
try:
|
||||
return int(index)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f'"{configured}" is not an NPU index; expected a number or "PCIe:<number>"'
|
||||
) from None
|
||||
|
||||
|
||||
def reinterpret(tensor: np.ndarray, byte_range: tuple[int, int], dtype) -> np.ndarray:
|
||||
"""Read a byte column range of a uint8 PPU tensor as `dtype`.
|
||||
|
||||
A column slice is not contiguous, and numpy will not reinterpret
|
||||
non-contiguous memory as a differently sized dtype, so it is copied first.
|
||||
"""
|
||||
"""Read a byte column range of a uint8 PPU tensor as `dtype`, copied
|
||||
first because numpy will not reinterpret non-contiguous memory."""
|
||||
lo, hi = byte_range
|
||||
return np.ascontiguousarray(tensor[:, lo:hi]).view(dtype)
|
||||
|
||||
@ -209,14 +319,9 @@ def _softmax(x: np.ndarray) -> np.ndarray:
|
||||
def _split_box_and_class_outputs(
|
||||
outputs: list[np.ndarray],
|
||||
) -> tuple[np.ndarray, np.ndarray] | None:
|
||||
"""Split a 2-tensor raw output into (box tensor, class tensor) by shape.
|
||||
|
||||
Used for model types where DX-COM's raw output tensor order isn't
|
||||
confirmed. Whichever tensor's shape reads as box-like -- (..., 4), or
|
||||
(..., 4, bins) for a per-side distribution -- is treated as boxes; the
|
||||
other is class scores. Returns None if that can't be told apart, which
|
||||
a class-score tensor with exactly 4 classes would also trigger.
|
||||
"""
|
||||
"""Split a 2-tensor raw output into (box tensor, class tensor) by shape,
|
||||
for model types where DX-COM's order is not confirmed: a box-like (..., 4)
|
||||
or (..., 4, bins) wins, None when ambiguous as with 4 classes."""
|
||||
if len(outputs) != 2:
|
||||
return None
|
||||
|
||||
@ -277,9 +382,8 @@ def run_nms(
|
||||
) -> np.ndarray:
|
||||
"""Return the indices surviving NMS, in descending score order."""
|
||||
if logger.isEnabledFor(logging.DEBUG):
|
||||
# Separates "the decode produced nothing" from "it produced boxes
|
||||
# whose scores all fell under the threshold", which look identical
|
||||
# from the empty detection array alone.
|
||||
# Separates "the decode produced nothing" from "every score fell under
|
||||
# the threshold", identical from the empty detection array alone.
|
||||
logger.debug(
|
||||
"NMS input: %d boxes, best score %.4f, %d over threshold %.2f",
|
||||
len(scores),
|
||||
@ -301,21 +405,17 @@ def run_nms(
|
||||
|
||||
def decode_ppu_anchor(
|
||||
outputs: list[np.ndarray],
|
||||
anchors: dict[int, np.ndarray],
|
||||
width: int,
|
||||
height: int,
|
||||
score_threshold: float,
|
||||
nms_threshold: float,
|
||||
) -> np.ndarray:
|
||||
"""Decode PPU output from an anchor-based head.
|
||||
"""Decode PPU output from an anchor-based head into Frigate's (20, 6) rows.
|
||||
|
||||
Models compiled with DEEPX's Post-Processing Unit perform candidate
|
||||
selection on the NPU and emit one fixed-width record per surviving
|
||||
detection, so only the anchor decode and NMS are left for the host. This
|
||||
is cheaper than decoding raw feature maps because every sub-threshold
|
||||
anchor has already been discarded.
|
||||
|
||||
Returns detections in Frigate's (20, 6) format of
|
||||
[class_id, score, y_min, x_min, y_max, x_max] with normalized coordinates.
|
||||
The NPU has already dropped sub-threshold candidates, leaving the anchor
|
||||
decode and NMS. A record names its anchor by index, so `anchors` has to be
|
||||
the table the model was trained with.
|
||||
"""
|
||||
records = ppu_records(outputs)
|
||||
|
||||
@ -332,16 +432,33 @@ def decode_ppu_anchor(
|
||||
anchor_idx = grid[:, 2]
|
||||
layer_idx = grid[:, 3]
|
||||
|
||||
# Both indices come off the wire and either could read past the tables
|
||||
# below, so a malformed record skips the frame rather than crashing.
|
||||
anchors_per_layer = len(next(iter(anchors.values())))
|
||||
|
||||
if np.any(layer_idx >= PPU_ANCHOR_LAYERS) or np.any(
|
||||
anchor_idx >= anchors_per_layer
|
||||
):
|
||||
logger.debug(
|
||||
"PPU record indexes layer %d / anchor %d, outside the %d layers and "
|
||||
"%d anchors per layer configured, skipping frame",
|
||||
layer_idx.max(),
|
||||
anchor_idx.max(),
|
||||
PPU_ANCHOR_LAYERS,
|
||||
anchors_per_layer,
|
||||
)
|
||||
return np.zeros((20, 6), np.float32)
|
||||
|
||||
stride = PPU_STRIDES[layer_idx]
|
||||
anchor_w = np.zeros(len(boxes), dtype=np.float32)
|
||||
anchor_h = np.zeros(len(boxes), dtype=np.float32)
|
||||
|
||||
for layer_stride, anchors in PPU_ANCHORS.items():
|
||||
for layer_stride, layer_anchors in anchors.items():
|
||||
mask = stride == layer_stride
|
||||
|
||||
if np.any(mask):
|
||||
anchor_w[mask] = anchors[anchor_idx[mask], 0]
|
||||
anchor_h[mask] = anchors[anchor_idx[mask], 1]
|
||||
anchor_w[mask] = layer_anchors[anchor_idx[mask], 0]
|
||||
anchor_h[mask] = layer_anchors[anchor_idx[mask], 1]
|
||||
|
||||
# Anchor-based decode: the model emits sigmoid outputs that are rescaled
|
||||
# against the grid cell and its anchor to recover pixel geometry.
|
||||
@ -375,13 +492,9 @@ def decode_ppu_anchor_free(
|
||||
score_threshold: float,
|
||||
nms_threshold: float,
|
||||
) -> np.ndarray:
|
||||
"""Decode PPU output from an anchor-free head.
|
||||
|
||||
These heads have no anchors and no objectness, so the PPU writes the box
|
||||
directly as (center_x, center_y, width, height) in input pixels. The grid
|
||||
columns an anchor-based record carries are unused here, and applying the
|
||||
anchor formula to them is what produces boxes hundreds of thousands of
|
||||
pixels wide.
|
||||
"""Decode PPU output from an anchor-free head: no anchors and no
|
||||
objectness, so the box arrives as (cx, cy, w, h) pixels. Running the
|
||||
anchor formula over the unused grid columns is what yields absurd boxes.
|
||||
"""
|
||||
records = ppu_records(outputs)
|
||||
|
||||
@ -419,13 +532,10 @@ def decode_raw_anchor(
|
||||
score_threshold: float,
|
||||
nms_threshold: float,
|
||||
) -> np.ndarray:
|
||||
"""Decode raw output from an anchor-based head.
|
||||
|
||||
The exported head emits (N, 5+C) rows of
|
||||
[center_x, center_y, width, height, objectness, class scores...] already in
|
||||
input pixels. Confidence is objectness times the best class score, which is
|
||||
what separates this layout from the anchor-free one: reading column 4 as a
|
||||
class score there would treat objectness as a class.
|
||||
"""Decode raw output from an anchor-based head: (N, 5+C) rows of
|
||||
[cx, cy, w, h, objectness, class scores...] in pixels. Confidence is
|
||||
objectness times the best class score, which is what parts this from the
|
||||
anchor-free layout, where column 4 is already a class.
|
||||
"""
|
||||
tensor = outputs[0]
|
||||
|
||||
@ -473,21 +583,14 @@ def decode_raw_anchor_free(
|
||||
) -> np.ndarray:
|
||||
"""Decode raw output from an anchor-free head, or an unconfirmed layout.
|
||||
|
||||
The exported head emits a single tensor with no objectness column,
|
||||
(4+C, N) channel-major or already transposed to (N, 4+C), and box columns
|
||||
already in input pixels as (center_x, center_y, width, height). This is
|
||||
also what `model_format: auto` falls back to: with only one output
|
||||
tensor there is no objectness column to tell an anchor-based head from
|
||||
an anchor-free one apart, so every column past the box is read as a
|
||||
class score, which is only correct for an anchor-free head -- matching
|
||||
`auto` being "only reliable for anchor-free heads" per its config
|
||||
description.
|
||||
One tensor with no objectness column, (4+C, N) or (N, 4+C), boxes as
|
||||
(cx, cy, w, h) pixels. `model_format: auto` lands here too, reading every
|
||||
column past the box as a class, which only holds for an anchor-free head.
|
||||
"""
|
||||
predictions = outputs[0]
|
||||
|
||||
# Drop the batch axis without a blind squeeze, which would also collapse
|
||||
# the box-count axis on the single-candidate frame case. Fold every
|
||||
# leading axis into rows instead.
|
||||
# the box count on a single-candidate frame. Fold leading axes into rows.
|
||||
if predictions.ndim == 3 and predictions.shape[0] == 1:
|
||||
predictions = predictions[0]
|
||||
|
||||
@ -532,11 +635,9 @@ def decode_raw_nms_in_head(
|
||||
height: int,
|
||||
score_threshold: float,
|
||||
) -> np.ndarray:
|
||||
"""Decode raw output from a head that already ran NMS.
|
||||
|
||||
These heads emit (N, 6) rows of [x_min, y_min, x_max, y_max, score, class]
|
||||
in input pixels, sorted by score. Running NMS again would be wasted work,
|
||||
so only the score filter and normalization are applied.
|
||||
"""Decode raw output from a head that already ran NMS: (N, 6) rows of
|
||||
[x_min, y_min, x_max, y_max, score, class] in pixels, sorted by score, so
|
||||
only the score filter and normalization are left.
|
||||
"""
|
||||
tensor = outputs[0]
|
||||
|
||||
@ -566,105 +667,10 @@ def decode_raw_nms_in_head(
|
||||
)
|
||||
|
||||
|
||||
def decode_ssd_raw(
|
||||
outputs: list[np.ndarray],
|
||||
width: int,
|
||||
height: int,
|
||||
score_threshold: float,
|
||||
nms_threshold: float,
|
||||
) -> np.ndarray:
|
||||
"""Decode SSD output: already-decoded boxes plus per-class probabilities.
|
||||
|
||||
DX-COM keeps SSD's postprocessing inside the compiled model. The .dxnn
|
||||
runs the NPU head into a small CPU subgraph that concatenates the
|
||||
per-feature-map outputs, applies softmax to the confidences, and decodes
|
||||
the prior-box regression (center variance 0.1, size variance 0.2, prior
|
||||
table baked in as initializers) all the way to corner-form boxes. That
|
||||
leaves only NMS for the host, which is what DEEPX's own ssdmv1,
|
||||
ssdmv2lite and ssdvgg16 demos do -- their postprocessor has no prior
|
||||
table, no variance constant, and no softmax.
|
||||
|
||||
Expects two output tensors, order-independent: boxes shaped (1, N, 4) as
|
||||
(x_min, y_min, x_max, y_max) normalized to [0, 1], and probabilities
|
||||
shaped (1, N, num_classes + 1) with background at class 0. Background is
|
||||
dropped and surviving labels are 0-indexed, to match a labelmap with no
|
||||
background row. N is whatever the backbone produces (3000 for the
|
||||
MobileNet variants, 8732 for VGG16) and never has to be known ahead of
|
||||
time.
|
||||
"""
|
||||
split = _split_box_and_class_outputs(outputs)
|
||||
|
||||
if split is None:
|
||||
logger.debug(
|
||||
"Could not identify SSD box/class outputs in %d tensor(s) with "
|
||||
"shapes %s, skipping frame",
|
||||
len(outputs),
|
||||
_shapes(outputs),
|
||||
)
|
||||
return np.zeros((20, 6), np.float32)
|
||||
|
||||
boxes, confidences = split
|
||||
|
||||
if (
|
||||
boxes.ndim != 3
|
||||
or confidences.ndim != 3
|
||||
or boxes.shape[0] == 0
|
||||
or boxes.shape[1] == 0
|
||||
):
|
||||
logger.debug(
|
||||
"Unexpected SSD output rank: boxes %s, classes %s, skipping frame",
|
||||
boxes.shape,
|
||||
confidences.shape,
|
||||
)
|
||||
return np.zeros((20, 6), np.float32)
|
||||
|
||||
boxes = boxes[0]
|
||||
confidences = confidences[0]
|
||||
|
||||
if boxes.shape[0] != confidences.shape[0]:
|
||||
logger.debug(
|
||||
"SSD box count %d does not match class count %d, skipping frame",
|
||||
boxes.shape[0],
|
||||
confidences.shape[0],
|
||||
)
|
||||
return np.zeros((20, 6), np.float32)
|
||||
|
||||
# the model already softmaxed these; drop background (class 0) and
|
||||
# relabel 0-indexed to match a labelmap with no background row
|
||||
scores = confidences[:, 1:]
|
||||
labels = np.argmax(scores, axis=1)
|
||||
scores = scores[np.arange(len(labels)), labels]
|
||||
|
||||
# normalized corner form -> input pixels, which is what NMS and
|
||||
# fill_detections work in
|
||||
x_min = boxes[:, 0] * width
|
||||
y_min = boxes[:, 1] * height
|
||||
x_max = boxes[:, 2] * width
|
||||
y_max = boxes[:, 3] * height
|
||||
|
||||
order = run_nms(
|
||||
x_min,
|
||||
y_min,
|
||||
x_max - x_min,
|
||||
y_max - y_min,
|
||||
scores,
|
||||
score_threshold,
|
||||
nms_threshold,
|
||||
)
|
||||
|
||||
return fill_detections(
|
||||
x_min, y_min, x_max, y_max, scores, labels, width, height, order
|
||||
)
|
||||
|
||||
|
||||
def _dfl_integral(distribution: np.ndarray) -> np.ndarray:
|
||||
"""Reduce a DFL distribution to an expected distance via weighted sum.
|
||||
|
||||
Ported from tinyvision/DAMO-YOLO's Integral module (Apache-2.0): each of
|
||||
the 4 box sides is predicted as a probability distribution over
|
||||
`reg_max + 1` discrete distance bins rather than a single regressed
|
||||
value; this recovers one scalar distance per side, in stride units.
|
||||
"""
|
||||
From DAMO-YOLO's Integral module (Apache-2.0): each box side spans
|
||||
`reg_max + 1` bins, reduced to one distance in stride units."""
|
||||
reg_max = distribution.shape[-1] - 1
|
||||
project = np.arange(reg_max + 1, dtype=np.float32)
|
||||
return distribution @ project
|
||||
@ -673,12 +679,8 @@ def _dfl_integral(distribution: np.ndarray) -> np.ndarray:
|
||||
def _damoyolo_center_priors(
|
||||
width: int, height: int, strides: tuple[int, ...]
|
||||
) -> np.ndarray:
|
||||
"""One (center_x, center_y, stride) row per grid cell across all scales,
|
||||
in pixel space at the model's own input resolution.
|
||||
|
||||
Ported from tinyvision/DAMO-YOLO's
|
||||
ZeroHead.get_single_level_center_priors (Apache-2.0).
|
||||
"""
|
||||
"""One (center_x, center_y, stride) row per grid cell across all scales, in
|
||||
pixels. From DAMO-YOLO's ZeroHead.get_single_level_center_priors."""
|
||||
rows = []
|
||||
|
||||
for stride in strides:
|
||||
@ -702,23 +704,11 @@ def decode_damoyolo_raw(
|
||||
) -> np.ndarray:
|
||||
"""Decode DAMO-YOLO's raw output: sigmoid class scores + a DFL box head.
|
||||
|
||||
Ported from tinyvision/DAMO-YOLO's ZeroHead (Apache-2.0). DAMO-YOLO is
|
||||
anchor-free like YOLOv8 and newer, but unlike them its box regression is
|
||||
a per-side Distribution Focal Loss histogram rather than a value already
|
||||
in pixels, so it needs its own decode instead of reusing
|
||||
decode_raw_anchor. Not yet verified against a real compiled model --
|
||||
see the two shape branches below for what DX-COM's export is assumed to
|
||||
produce, and adjust if a real .dxnn's shapes don't match either one.
|
||||
|
||||
Expects two output tensors, order-independent: class scores shaped
|
||||
(1, N, num_classes), already sigmoid-activated, and a box output that is
|
||||
either (1, N, 4, reg_max + 1) -- a raw DFL distribution per side, decoded
|
||||
here via softmax + integral + stride-scaled distance-to-box -- or
|
||||
already (1, N, 4), in case DX-COM's export folds that whole decode in
|
||||
(the reference only ever produces a plain 4-wide box tensor as the
|
||||
*result* of that decode, so this shape is treated as final
|
||||
(x_min, y_min, x_max, y_max) pixel coordinates, not as a second
|
||||
distance representation).
|
||||
From DAMO-YOLO's ZeroHead (Apache-2.0). Anchor-free, but boxes are per-side
|
||||
DFL histograms, not the pixel distances decode_raw_anchor_free expects.
|
||||
Takes two tensors in either order: (1, N, num_classes) sigmoid scores, and
|
||||
boxes as (1, N, 4, reg_max + 1) raw DFL decoded here, or (1, N, 4) pixel
|
||||
corners should DX-COM fold it in. Unverified against a compiled model.
|
||||
"""
|
||||
split = _split_box_and_class_outputs(outputs)
|
||||
|
||||
@ -740,9 +730,8 @@ def decode_damoyolo_raw(
|
||||
num_priors = cls_scores.shape[0]
|
||||
|
||||
if box_output.ndim == 4:
|
||||
# raw per-side DFL distribution: softmax the bins, reduce to an
|
||||
# expected distance (in stride units), then scale by each prior's
|
||||
# own stride and apply it against that prior's center
|
||||
# raw per-side DFL: softmax the bins, reduce to a distance in stride
|
||||
# units, then scale by each prior's stride against its own center
|
||||
distances = _dfl_integral(_softmax(box_output[0]))
|
||||
|
||||
if distances.shape[0] != num_priors:
|
||||
@ -819,13 +808,11 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
model_config = ConfigDict(title="DEEPX NPU")
|
||||
|
||||
type: Literal[DETECTOR_KEY]
|
||||
device_ids: str = Field(
|
||||
default=DEVICE_IDS_AUTO,
|
||||
title="DEEPX device indices",
|
||||
description="Comma-separated NPU device indices this detector should "
|
||||
'bind to, e.g. "0,1". Leave as "auto" to detect them from the '
|
||||
"DXNN_DEVICES environment variable, then the DX-RT device count, then "
|
||||
"a scan of /dev/dxrt*.",
|
||||
device: str = Field(
|
||||
default="",
|
||||
title="DEEPX device",
|
||||
description="Which NPU this detector binds to, as PCIe:<index>. "
|
||||
"Empty selects the first NPU.",
|
||||
)
|
||||
model_type: DeepxModelTypeEnum = Field(
|
||||
default=DeepxModelTypeEnum.yologeneric,
|
||||
@ -850,8 +837,18 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
"from the output shape, which cannot tell an anchor-based head from "
|
||||
"an anchor-free one and is only reliable for anchor-free heads. "
|
||||
"Required when ppu is true, since a PPU record's layout is identical "
|
||||
"for every yolo-generic variant and cannot be inferred. ssd and "
|
||||
"damo-yolo each read one fixed layout and take no value here.",
|
||||
"for every yolo-generic variant and cannot be inferred. damo-yolo "
|
||||
"reads one fixed layout and takes no value here.",
|
||||
)
|
||||
anchors: str = Field(
|
||||
default="",
|
||||
title="Anchor sizes an anchor-based PPU model was trained with",
|
||||
description="The model's own anchor table, as one group of "
|
||||
"comma-separated width,height pairs per feature-map layer, groups "
|
||||
'separated by ";" in stride order 8, 16, 32. Required when ppu is '
|
||||
"true and model_format is anchor, and not used otherwise: a PPU "
|
||||
"record names its anchor by index only, so the table has to come "
|
||||
"from the model. Copy it from the config the model was trained with.",
|
||||
)
|
||||
score_threshold: float = Field(
|
||||
default=0.25,
|
||||
@ -862,34 +859,40 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
title="NMS IoU threshold used when decoding detector output",
|
||||
)
|
||||
|
||||
@field_validator("device_ids", mode="before")
|
||||
@field_validator("device")
|
||||
@classmethod
|
||||
def coerce_device_ids(cls, value):
|
||||
"""Accept the yaml list form as well as the comma-separated string
|
||||
the config form writes, since a bare list is the natural thing to
|
||||
hand-write. A blank entry folds back to the auto sentinel."""
|
||||
if isinstance(value, (list, tuple)):
|
||||
value = ",".join(str(item) for item in value)
|
||||
elif isinstance(value, int) and not isinstance(value, bool):
|
||||
value = str(value)
|
||||
def validate_device(cls, value: str) -> str:
|
||||
"""Reject an unusable device at startup rather than at first inference."""
|
||||
resolve_device(value)
|
||||
return value
|
||||
|
||||
if value is None or (isinstance(value, str) and not value.strip()):
|
||||
return DEVICE_IDS_AUTO
|
||||
@field_validator("anchors", mode="before")
|
||||
@classmethod
|
||||
def coerce_anchors(cls, value):
|
||||
"""Accept the nested list a model's config declares anchors in, flat
|
||||
row or explicit pairs, as well as the string the form writes."""
|
||||
if value is None:
|
||||
return ""
|
||||
|
||||
if isinstance(value, (list, tuple)):
|
||||
return ";".join(
|
||||
",".join(str(number) for number in _flatten(group)) for group in value
|
||||
)
|
||||
|
||||
return value
|
||||
|
||||
@field_validator("device_ids")
|
||||
@field_validator("anchors")
|
||||
@classmethod
|
||||
def validate_device_ids(cls, value: str) -> str:
|
||||
"""Reject a device list that can't be parsed, so the error surfaces at
|
||||
config load rather than when the detector process starts."""
|
||||
def validate_anchors(cls, value: str) -> str:
|
||||
"""Reject an anchor table that can't be read, so the error surfaces at
|
||||
config load rather than on the first frame."""
|
||||
if not value.strip():
|
||||
return ""
|
||||
|
||||
try:
|
||||
parse_device_ids(value)
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
f'device_ids "{value}" is not a comma-separated list of NPU '
|
||||
f'device indices, e.g. "0,1", or "{DEVICE_IDS_AUTO}".'
|
||||
) from None
|
||||
parse_anchors(value)
|
||||
except ValueError as err:
|
||||
raise ValueError(f"anchors could not be read: {err}") from None
|
||||
|
||||
return value
|
||||
|
||||
@ -925,14 +928,10 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_ppu_requires_model_format(self):
|
||||
"""Reject an unset model_format when ppu is enabled.
|
||||
|
||||
A PPU record is the same fixed-width layout for every yolo-generic
|
||||
variant, so unlike the raw output path there is no shape to infer
|
||||
the layout from. Leaving model_format unset here used to silently
|
||||
default to the anchor-based decoder, which corrupts detections from
|
||||
anchor-free PPU models instead of failing loudly.
|
||||
"""
|
||||
"""Reject an unset model_format when ppu is enabled. Every
|
||||
yolo-generic variant shares one PPU record layout, so there is no
|
||||
shape to infer from, and defaulting to the anchor-based decoder
|
||||
corrupts anchor-free detections rather than failing loudly."""
|
||||
if self.ppu and self.model_format is ModelFormatEnum.auto:
|
||||
raise ValueError(
|
||||
"model_format must be set when ppu is true. The PPU output "
|
||||
@ -943,12 +942,35 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_model_format_matches_model_type(self):
|
||||
"""Reject a model_format on a model_type that has no layout to pick.
|
||||
def validate_anchors_match_the_decode_path(self):
|
||||
"""Require an anchor table for the one decode that reads it, and
|
||||
reject one anywhere else. A record names its anchor by index only and
|
||||
the table behind it differs per model, so a built-in default would
|
||||
silently rescale every box. Refuse to guess."""
|
||||
reads_anchors = self.ppu and self.model_format in ANCHOR_FORMATS
|
||||
|
||||
Only yolo-generic has variants the decoder can't tell apart. SSD and
|
||||
DAMO-YOLO each read one fixed output layout, so any value other than
|
||||
the sentinel is a config mistake rather than a choice."""
|
||||
if reads_anchors and not self.anchors:
|
||||
raise ValueError(
|
||||
"anchors must be set when ppu is true and model_format is "
|
||||
f"'{self.model_format.value}'. Copy the anchor table from the "
|
||||
"config the model was trained with, one group per feature-map "
|
||||
'layer separated by ";", in stride order 8, 16, 32.'
|
||||
)
|
||||
|
||||
if self.anchors and not reads_anchors:
|
||||
raise ValueError(
|
||||
"anchors is only read when ppu is true and model_format is "
|
||||
f"'{ModelFormatEnum.anchor.value}'; every other decode path "
|
||||
"takes its box geometry from the model output. Remove it."
|
||||
)
|
||||
|
||||
return self
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_model_format_matches_model_type(self):
|
||||
"""Reject a model_format on a model_type that has no layout to pick:
|
||||
only yolo-generic has variants the decoder cannot tell apart, so a
|
||||
value on DAMO-YOLO's one fixed layout is a mistake, not a choice."""
|
||||
if self.model_type is DeepxModelTypeEnum.yologeneric:
|
||||
return self
|
||||
|
||||
@ -962,28 +984,34 @@ class DeepxDetectorConfig(BaseDetectorConfig):
|
||||
return self
|
||||
|
||||
|
||||
class Deepx(DetectionApi):
|
||||
class DeepxDetector(DetectionApi):
|
||||
type_key = DETECTOR_KEY
|
||||
runtime_manifest = DEEPX_MANIFEST
|
||||
|
||||
def __init__(self, config: DeepxDetectorConfig):
|
||||
self.activate_dependencies()
|
||||
|
||||
# DX-RT tries an abstract socket first, which never crosses a network
|
||||
# namespace, and only falls back after half a second of retries.
|
||||
# Naming the filesystem one up front skips that; dxrtd listens on both.
|
||||
os.environ.setdefault(DXRT_IPC_ENDPOINT_ENV, DXRT_IPC_SOCKET)
|
||||
satisfy_service_check(os.environ[DXRT_IPC_ENDPOINT_ENV])
|
||||
|
||||
try:
|
||||
from dx_engine import InferenceEngine, InferenceOption
|
||||
except ImportError as err:
|
||||
except ModuleNotFoundError:
|
||||
raise ImportError(
|
||||
"The DX-RT runtime is not available. Check that the DEEPX "
|
||||
"kernel driver is loaded on the Docker host and that "
|
||||
"/dev/dxrt* is passed through to this container."
|
||||
) from err
|
||||
"The DX-RT python bindings are not installed. Frigate installs "
|
||||
"them at startup when a DEEPX detector is configured; check the "
|
||||
"startup log for errors."
|
||||
) from None
|
||||
|
||||
super().__init__(config)
|
||||
|
||||
# Frigate ignores a nested `model` block, so model.model_type carries
|
||||
# the root value -- `ssd` whenever it was left unset. The detector's
|
||||
# own model_type is the one the user actually picked, so it wins, and
|
||||
# is written back for everything downstream that reads the model
|
||||
# config (the events table, /api/config).
|
||||
self.model_type = ModelTypeEnum(config.model_type.value)
|
||||
config.model.model_type = self.model_type
|
||||
# In 0.19 model_type sits on the model, not the detector: a device
|
||||
# string fills only the device field, so DeepxDetectorConfig.model_type
|
||||
# never reflects what the user configured and must not be read here.
|
||||
self.model_type = ModelTypeEnum(config.model.model_type.value)
|
||||
|
||||
model_path = config.model.path
|
||||
|
||||
@ -994,16 +1022,18 @@ class Deepx(DetectionApi):
|
||||
"model.path at the .dxnn file."
|
||||
)
|
||||
|
||||
devices = resolve_devices(config.device_ids)
|
||||
logger.info("Loading DEEPX model %s on device(s) %s", model_path, devices)
|
||||
device = resolve_device(config.device)
|
||||
logger.info("Loading DEEPX model %s on device %s", model_path, device)
|
||||
|
||||
options = InferenceOption()
|
||||
options.devices = devices
|
||||
options.devices = [device]
|
||||
options.bound_option = InferenceOption.BOUND_OPTION.NPU_ALL
|
||||
|
||||
self.session = InferenceEngine(str(model_path), options)
|
||||
self.ppu = config.ppu
|
||||
self.model_format = config.model_format
|
||||
# validated above, and only non-empty for the decode that reads it
|
||||
self.anchors = parse_anchors(config.anchors) if config.anchors else {}
|
||||
self.score_threshold = config.score_threshold
|
||||
self.nms_threshold = config.nms_threshold
|
||||
|
||||
@ -1021,15 +1051,6 @@ class Deepx(DetectionApi):
|
||||
def decode(self, outputs: list[np.ndarray]) -> np.ndarray:
|
||||
"""Decode raw or PPU output according to the configured model type
|
||||
and detection head layout."""
|
||||
if self.model_type == ModelTypeEnum.ssd:
|
||||
return decode_ssd_raw(
|
||||
outputs,
|
||||
self.width,
|
||||
self.height,
|
||||
self.score_threshold,
|
||||
self.nms_threshold,
|
||||
)
|
||||
|
||||
if self.model_type == ModelTypeEnum.damoyolo:
|
||||
return decode_damoyolo_raw(
|
||||
outputs,
|
||||
@ -1041,9 +1062,8 @@ class Deepx(DetectionApi):
|
||||
|
||||
# model_type is yolo-generic from here on
|
||||
if self.ppu:
|
||||
# Config validation guarantees model_format is set to an anchor
|
||||
# or anchor-free format whenever ppu is true, since PPU records
|
||||
# give no other way to tell the two apart.
|
||||
# Config validation guarantees an anchor or anchor-free format
|
||||
# whenever ppu is true; PPU records give no other way to tell.
|
||||
if self.model_format in ANCHOR_FREE_FORMATS:
|
||||
return decode_ppu_anchor_free(
|
||||
outputs,
|
||||
@ -1055,6 +1075,7 @@ class Deepx(DetectionApi):
|
||||
|
||||
return decode_ppu_anchor(
|
||||
outputs,
|
||||
self.anchors,
|
||||
self.width,
|
||||
self.height,
|
||||
self.score_threshold,
|
||||
@ -1066,9 +1087,8 @@ class Deepx(DetectionApi):
|
||||
outputs, self.width, self.height, self.score_threshold
|
||||
)
|
||||
|
||||
# Only single-output anchor heads need the dedicated decoder. A
|
||||
# multi-part output is a set of raw feature maps, which the shared
|
||||
# helper already decodes with the same anchor formula.
|
||||
# Only single-output anchor heads need the dedicated decoder; a
|
||||
# multi-part output is feature maps the shared helper already reads.
|
||||
if self.model_format in ANCHOR_FORMATS and len(outputs) == 1:
|
||||
return decode_raw_anchor(
|
||||
outputs,
|
||||
@ -1078,10 +1098,9 @@ class Deepx(DetectionApi):
|
||||
self.nms_threshold,
|
||||
)
|
||||
|
||||
# `auto` (the default) and an explicit `anchor_free` both read a
|
||||
# single tensor with no objectness column. Routing these through the
|
||||
# dedicated decoder keeps them honoring this detector's own
|
||||
# score_threshold and nms_threshold
|
||||
# `auto` and an explicit `anchor_free` both read one tensor with no
|
||||
# objectness column, and the dedicated decoder keeps them honoring
|
||||
# this detector's own score_threshold and nms_threshold
|
||||
if len(outputs) == 1 and (
|
||||
self.model_format in ANCHOR_FREE_FORMATS
|
||||
or self.model_format is ModelFormatEnum.auto
|
||||
@ -1121,12 +1140,8 @@ class Deepx(DetectionApi):
|
||||
def log_layout(self, tensor_input, outputs: list[np.ndarray]) -> None:
|
||||
"""Log the model's real input and output layout, once per start.
|
||||
|
||||
Every decode path is a guess about the layout DX-COM compiled the
|
||||
detection head into, and a wrong guess returns an empty detection
|
||||
array rather than failing. These shapes and value ranges are the
|
||||
first thing needed to tell those apart, so they go out at info
|
||||
level: waiting to reproduce them with debug logging enabled costs a
|
||||
restart, and this is one line per tensor per detector start.
|
||||
A wrong guess at the compiled layout returns an empty detection array
|
||||
rather than failing, and these shapes are what tell those apart.
|
||||
"""
|
||||
tensors = [("input", tensor_input)]
|
||||
tensors += [(f"output[{i}]", out) for i, out in enumerate(outputs)]
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
"""Tests for the DEEPX detector's per-format output decoding."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
from pydantic import ValidationError
|
||||
@ -9,10 +12,14 @@ from frigate.detectors.detector_config import ModelTypeEnum
|
||||
from frigate.detectors.plugins.deepx import (
|
||||
ANCHOR_FORMATS,
|
||||
ANCHOR_FREE_FORMATS,
|
||||
DEVICE_IDS_AUTO,
|
||||
DEEPX_MANIFEST,
|
||||
DXRT_IPC_ENDPOINT_ENV,
|
||||
DXRT_IPC_SOCKET,
|
||||
DXRT_VERSION,
|
||||
NMS_IN_HEAD_FORMATS,
|
||||
PPU_ANCHORS,
|
||||
PPU_ANCHOR_LAYERS,
|
||||
PPU_RECORD_SIZE,
|
||||
DeepxDetector,
|
||||
DeepxDetectorConfig,
|
||||
DeepxModelTypeEnum,
|
||||
ModelFormatEnum,
|
||||
@ -22,10 +29,17 @@ from frigate.detectors.plugins.deepx import (
|
||||
decode_raw_anchor,
|
||||
decode_raw_anchor_free,
|
||||
decode_raw_nms_in_head,
|
||||
decode_ssd_raw,
|
||||
parse_device_ids,
|
||||
resolve_devices,
|
||||
parse_anchors,
|
||||
resolve_device,
|
||||
satisfy_service_check,
|
||||
service_is_visible,
|
||||
)
|
||||
from frigate.util.runtime_deps import ArtifactKind
|
||||
|
||||
# Deliberately not any published model's anchor table: a decoder that fell back
|
||||
# to a built-in set instead of reading the configured one would not match these.
|
||||
TEST_ANCHOR_STRING = "11,17,23,29,31,37;41,47,53,59,61,67;71,73,79,83,89,97"
|
||||
TEST_ANCHORS = parse_anchors(TEST_ANCHOR_STRING)
|
||||
|
||||
|
||||
def build_ppu_record(
|
||||
@ -61,9 +75,9 @@ class TestDeepxPpuDecode(unittest.TestCase):
|
||||
).reshape(1, 1, PPU_RECORD_SIZE)
|
||||
]
|
||||
|
||||
detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
|
||||
detections = decode_ppu_anchor(outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45)
|
||||
|
||||
anchor_w, anchor_h = PPU_ANCHORS[8][0]
|
||||
anchor_w, anchor_h = TEST_ANCHORS[8][0]
|
||||
# center = (0.5 * 2 - 0.5 + 10) * 8, size = (0.5 ** 2 * 4) * anchor
|
||||
center = (0.5 * 2.0 - 0.5 + 10) * 8
|
||||
box_w = 0.5**2 * 4.0 * anchor_w
|
||||
@ -93,7 +107,7 @@ class TestDeepxPpuDecode(unittest.TestCase):
|
||||
label=0,
|
||||
).reshape(1, 1, PPU_RECORD_SIZE)
|
||||
]
|
||||
return decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
|
||||
return decode_ppu_anchor(outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45)
|
||||
|
||||
small = decode(0)
|
||||
large = decode(2)
|
||||
@ -116,7 +130,7 @@ class TestDeepxPpuDecode(unittest.TestCase):
|
||||
).reshape(1, 1, PPU_RECORD_SIZE)
|
||||
]
|
||||
|
||||
detections = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
|
||||
detections = decode_ppu_anchor(outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45)
|
||||
|
||||
self.assertTrue(np.all(detections == 0))
|
||||
|
||||
@ -137,19 +151,150 @@ class TestDeepxPpuDecode(unittest.TestCase):
|
||||
)
|
||||
|
||||
detections = decode_ppu_anchor(
|
||||
[records.reshape(1, 40, PPU_RECORD_SIZE)], 640, 640, 0.25, 0.45
|
||||
[records.reshape(1, 40, PPU_RECORD_SIZE)],
|
||||
TEST_ANCHORS,
|
||||
640,
|
||||
640,
|
||||
0.25,
|
||||
0.45,
|
||||
)
|
||||
|
||||
self.assertEqual(detections.shape, (20, 6))
|
||||
|
||||
def test_returns_empty_detections_for_no_output(self):
|
||||
self.assertTrue(np.all(decode_ppu_anchor([], 640, 640, 0.25, 0.45) == 0))
|
||||
self.assertTrue(
|
||||
np.all(decode_ppu_anchor([], TEST_ANCHORS, 640, 640, 0.25, 0.45) == 0)
|
||||
)
|
||||
|
||||
def test_returns_empty_detections_for_an_unexpected_record_width(self):
|
||||
# a non-PPU model would produce a different record width
|
||||
outputs = [np.zeros((1, 4, 16), dtype=np.uint8)]
|
||||
|
||||
self.assertTrue(np.all(decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45) == 0))
|
||||
self.assertTrue(
|
||||
np.all(decode_ppu_anchor(outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45) == 0)
|
||||
)
|
||||
|
||||
|
||||
class TestDeepxAnchors(unittest.TestCase):
|
||||
"""The anchor table comes from the model, not from this file. A PPU record
|
||||
names its anchor by index alone, and that index means different box sizes
|
||||
for different models, so there is nothing safe to default to."""
|
||||
|
||||
def test_parses_a_group_per_stride_keyed_by_stride(self):
|
||||
table = parse_anchors("11,17,23,29;41,47,53,59;71,73,79,83")
|
||||
|
||||
self.assertEqual(sorted(table), [8, 16, 32])
|
||||
np.testing.assert_array_equal(table[8], [[11, 17], [23, 29]])
|
||||
np.testing.assert_array_equal(table[32], [[71, 73], [79, 83]])
|
||||
|
||||
def test_accepts_the_nested_list_a_model_config_declares(self):
|
||||
config = DeepxDetectorConfig(
|
||||
type="deepx",
|
||||
ppu=True,
|
||||
model_format=ModelFormatEnum.anchor,
|
||||
anchors=[[11, 17, 23, 29], [41, 47, 53, 59], [71, 73, 79, 83]],
|
||||
)
|
||||
|
||||
np.testing.assert_array_equal(
|
||||
parse_anchors(config.anchors)[8], [[11, 17], [23, 29]]
|
||||
)
|
||||
|
||||
def test_accepts_explicit_pairs(self):
|
||||
config = DeepxDetectorConfig(
|
||||
type="deepx",
|
||||
ppu=True,
|
||||
model_format=ModelFormatEnum.anchor,
|
||||
anchors=[[[11, 17], [23, 29]], [[41, 47], [53, 59]], [[71, 73], [79, 83]]],
|
||||
)
|
||||
|
||||
np.testing.assert_array_equal(
|
||||
parse_anchors(config.anchors)[8], [[11, 17], [23, 29]]
|
||||
)
|
||||
|
||||
def test_anchor_ppu_without_anchors_is_rejected(self):
|
||||
"""Guessing the table silently rescales every box, so refuse to."""
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
type="deepx", ppu=True, model_format=ModelFormatEnum.anchor
|
||||
)
|
||||
|
||||
def test_anchors_on_a_decode_that_ignores_them_is_rejected(self):
|
||||
for kwargs in (
|
||||
dict(ppu=True, model_format=ModelFormatEnum.anchor_free),
|
||||
dict(model_format=ModelFormatEnum.anchor),
|
||||
dict(),
|
||||
):
|
||||
with self.subTest(**kwargs):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
type="deepx", anchors=TEST_ANCHOR_STRING, **kwargs
|
||||
)
|
||||
|
||||
def test_a_group_per_stride_is_required(self):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
type="deepx",
|
||||
ppu=True,
|
||||
model_format=ModelFormatEnum.anchor,
|
||||
anchors="11,17;41,47",
|
||||
)
|
||||
|
||||
def test_groups_must_hold_equal_length_pairs(self):
|
||||
for bad in ("11,17,23;41,47,53;71,73,79", "11,17;41,47,53,59;71,73"):
|
||||
with self.subTest(anchors=bad):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
type="deepx",
|
||||
ppu=True,
|
||||
model_format=ModelFormatEnum.anchor,
|
||||
anchors=bad,
|
||||
)
|
||||
|
||||
def test_decode_uses_the_supplied_table(self):
|
||||
"""Two tables over one record must give two different box sizes."""
|
||||
outputs = [
|
||||
build_ppu_record(
|
||||
box=(0.5, 0.5, 0.5, 0.5),
|
||||
grid_y=10,
|
||||
grid_x=10,
|
||||
anchor_idx=0,
|
||||
layer_idx=0,
|
||||
score=0.9,
|
||||
label=2,
|
||||
).reshape(1, 1, PPU_RECORD_SIZE)
|
||||
]
|
||||
|
||||
narrow = decode_ppu_anchor(
|
||||
outputs, parse_anchors("4,4;8,8;16,16"), 640, 640, 0.25, 0.45
|
||||
)
|
||||
wide = decode_ppu_anchor(
|
||||
outputs, parse_anchors("40,40;80,80;160,160"), 640, 640, 0.25, 0.45
|
||||
)
|
||||
|
||||
self.assertGreater(wide[0][5] - wide[0][3], narrow[0][5] - narrow[0][3])
|
||||
|
||||
def test_an_out_of_range_index_skips_the_frame(self):
|
||||
"""Both indices come off the wire, and reading past the table would
|
||||
take the detection process down rather than drop one frame."""
|
||||
for anchor_idx, layer_idx in ((9, 0), (0, PPU_ANCHOR_LAYERS)):
|
||||
with self.subTest(anchor_idx=anchor_idx, layer_idx=layer_idx):
|
||||
outputs = [
|
||||
build_ppu_record(
|
||||
box=(0.5, 0.5, 0.5, 0.5),
|
||||
grid_y=10,
|
||||
grid_x=10,
|
||||
anchor_idx=anchor_idx,
|
||||
layer_idx=layer_idx,
|
||||
score=0.9,
|
||||
label=2,
|
||||
).reshape(1, 1, PPU_RECORD_SIZE)
|
||||
]
|
||||
|
||||
detections = decode_ppu_anchor(
|
||||
outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45
|
||||
)
|
||||
|
||||
self.assertTrue(np.all(detections == 0))
|
||||
|
||||
|
||||
class TestDeepxModelFormats(unittest.TestCase):
|
||||
@ -231,21 +376,6 @@ class TestDeepxDetectorConfig(unittest.TestCase):
|
||||
model_type=DeepxModelTypeEnum.damoyolo,
|
||||
)
|
||||
|
||||
def test_ssd_model_type_takes_no_model_format(self):
|
||||
"""SSD's layout is fixed: DX-COM keeps the softmax and prior decode
|
||||
inside the .dxnn, so no backbone has to be named."""
|
||||
config = DeepxDetectorConfig(type="deepx", model_type=DeepxModelTypeEnum.ssd)
|
||||
|
||||
self.assertIs(config.model_format, ModelFormatEnum.auto)
|
||||
|
||||
def test_ssd_model_type_rejects_a_model_format(self):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
type="deepx",
|
||||
model_format=ModelFormatEnum.anchor_free,
|
||||
model_type=DeepxModelTypeEnum.ssd,
|
||||
)
|
||||
|
||||
def test_damoyolo_model_type_rejects_a_model_format(self):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(
|
||||
@ -263,55 +393,66 @@ class TestDeepxDetectorConfig(unittest.TestCase):
|
||||
self.assertIs(config.model_format, ModelFormatEnum.auto)
|
||||
|
||||
|
||||
class TestDeepxDeviceIds(unittest.TestCase):
|
||||
def test_a_comma_separated_string_is_kept(self):
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids="0,1")
|
||||
class TestDeepxDeviceSelection(unittest.TestCase):
|
||||
def test_a_pcie_device_string_resolves_to_its_index(self):
|
||||
self.assertEqual(resolve_device("PCIe:1"), 1)
|
||||
|
||||
self.assertEqual(config.device_ids, "0,1")
|
||||
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
|
||||
def test_a_bare_index_is_accepted(self):
|
||||
self.assertEqual(resolve_device("2"), 2)
|
||||
|
||||
def test_a_yaml_list_is_accepted(self):
|
||||
"""A bare list is the natural thing to hand-write, so it is folded
|
||||
into the same comma-separated form the config UI writes."""
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids=[0, 1])
|
||||
def test_an_empty_device_is_the_first_npu(self):
|
||||
self.assertEqual(resolve_device(""), 0)
|
||||
|
||||
self.assertEqual(config.device_ids, "0,1")
|
||||
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
|
||||
def test_a_non_numeric_device_is_rejected(self):
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_device("PCIe:the-fast-one")
|
||||
|
||||
def test_a_single_index_is_accepted(self):
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids=1)
|
||||
|
||||
self.assertEqual(config.device_ids, "1")
|
||||
self.assertEqual(resolve_devices(config.device_ids), [1])
|
||||
|
||||
def test_surrounding_whitespace_is_tolerated(self):
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids=" 0 , 1 ")
|
||||
|
||||
self.assertEqual(resolve_devices(config.device_ids), [0, 1])
|
||||
|
||||
def test_an_unparsable_value_is_rejected_at_config_load(self):
|
||||
def test_a_comma_separated_list_is_no_longer_accepted(self):
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(type="deepx", device_ids="gpu0")
|
||||
DeepxDetectorConfig(type="deepx", device="0,1")
|
||||
|
||||
def test_unset_leaves_device_selection_to_auto_detection(self):
|
||||
config = DeepxDetectorConfig(type="deepx")
|
||||
def test_the_old_multi_device_field_is_gone(self):
|
||||
self.assertNotIn("device_ids", DeepxDetectorConfig.model_fields)
|
||||
|
||||
self.assertEqual(config.device_ids, DEVICE_IDS_AUTO)
|
||||
self.assertEqual(parse_device_ids(config.device_ids), [])
|
||||
def test_the_device_string_lands_on_the_device_field(self):
|
||||
# inherited from BaseDetectorConfig; asserted so a rename is caught here
|
||||
self.assertEqual(DeepxDetectorConfig.device_spec_field, "device")
|
||||
self.assertIn("device", DeepxDetectorConfig.model_fields)
|
||||
|
||||
def test_a_blank_entry_folds_to_auto(self):
|
||||
"""The config form round-trips a value for every field, so a blank or
|
||||
null entry has to land on the sentinel rather than stay None."""
|
||||
for value in (None, "", " ", []):
|
||||
with self.subTest(value=value):
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids=value)
|
||||
|
||||
self.assertEqual(config.device_ids, DEVICE_IDS_AUTO)
|
||||
class TestDeepxRuntimeManifest(unittest.TestCase):
|
||||
def test_the_detector_declares_a_manifest(self):
|
||||
self.assertIs(DeepxDetector.runtime_manifest, DEEPX_MANIFEST)
|
||||
|
||||
def test_auto_is_case_insensitive(self):
|
||||
config = DeepxDetectorConfig(type="deepx", device_ids="AUTO")
|
||||
def test_one_wheel_per_supported_machine(self):
|
||||
machines = sorted(m for a in DEEPX_MANIFEST.artifacts for m in a.machines)
|
||||
self.assertEqual(machines, ["aarch64", "x86_64"])
|
||||
|
||||
self.assertEqual(parse_device_ids(config.device_ids), [])
|
||||
def test_every_artifact_is_a_pinned_wheel(self):
|
||||
for artifact in DEEPX_MANIFEST.artifacts:
|
||||
with self.subTest(url=artifact.url):
|
||||
self.assertEqual(artifact.kind, ArtifactKind.wheel)
|
||||
self.assertEqual(len(artifact.sha256), 64)
|
||||
self.assertTrue(artifact.url.endswith(".whl"))
|
||||
|
||||
def test_the_wheels_match_the_container_interpreter(self):
|
||||
for artifact in DEEPX_MANIFEST.artifacts:
|
||||
with self.subTest(url=artifact.url):
|
||||
self.assertIn("cp311", artifact.url)
|
||||
|
||||
def test_every_url_carries_the_pinned_version(self):
|
||||
"""A PyPI path holds a per-file digest, so a version bump has to rewrite
|
||||
the whole URL rather than only the version constant."""
|
||||
self.assertEqual(DEEPX_MANIFEST.version, DXRT_VERSION)
|
||||
|
||||
for artifact in DEEPX_MANIFEST.artifacts:
|
||||
with self.subTest(url=artifact.url):
|
||||
self.assertIn(f"dx_engine-{DXRT_VERSION}-", artifact.url)
|
||||
|
||||
def test_no_library_preloading_is_needed(self):
|
||||
# the wheel is auditwheel-repaired and resolves its own libs by RPATH
|
||||
self.assertEqual(DEEPX_MANIFEST.preload, ())
|
||||
self.assertFalse(DEEPX_MANIFEST.needs_ld_library_path)
|
||||
|
||||
|
||||
class TestDeepxPpuAnchorFreeDecode(unittest.TestCase):
|
||||
@ -352,7 +493,7 @@ class TestDeepxPpuAnchorFreeDecode(unittest.TestCase):
|
||||
outputs = [record.reshape(1, 1, PPU_RECORD_SIZE)]
|
||||
|
||||
anchor_free = decode_ppu_anchor_free(outputs, 640, 640, 0.25, 0.45)
|
||||
anchor = decode_ppu_anchor(outputs, 640, 640, 0.25, 0.45)
|
||||
anchor = decode_ppu_anchor(outputs, TEST_ANCHORS, 640, 640, 0.25, 0.45)
|
||||
|
||||
# the anchor decode squares the width, saturating the clip to the frame
|
||||
self.assertEqual(anchor[0][5], 1.0)
|
||||
@ -480,7 +621,9 @@ class TestDeepxRawAnchorFreeDecode(unittest.TestCase):
|
||||
def test_returns_empty_detections_for_no_output(self):
|
||||
out = np.zeros((1, 0, 84), dtype=np.float32)
|
||||
|
||||
self.assertTrue(np.all(decode_raw_anchor_free([out], 640, 640, 0.25, 0.45) == 0))
|
||||
self.assertTrue(
|
||||
np.all(decode_raw_anchor_free([out], 640, 640, 0.25, 0.45) == 0)
|
||||
)
|
||||
|
||||
|
||||
class TestDeepxRawNmsInHeadDecode(unittest.TestCase):
|
||||
@ -526,110 +669,6 @@ class TestDeepxRawNmsInHeadDecode(unittest.TestCase):
|
||||
self.assertTrue(np.all(decode_raw_nms_in_head([out], 640, 640, 0.25) == 0))
|
||||
|
||||
|
||||
class TestDeepxSsdDecode(unittest.TestCase):
|
||||
"""The .dxnn emits final boxes and probabilities, so these feed the
|
||||
decoder what a real compiled SSD produces: corner-form boxes normalized
|
||||
to [0, 1] and post-softmax confidences with background at column 0."""
|
||||
|
||||
def build_ssd_output(self, box_index, box, class_id, prob, num_boxes=3000):
|
||||
num_classes = 21 # 20 VOC classes + background
|
||||
|
||||
boxes = np.zeros((1, num_boxes, 4), dtype=np.float32)
|
||||
boxes[0, box_index] = box
|
||||
|
||||
# every other box is background, as the model would report it
|
||||
scores = np.zeros((1, num_boxes, num_classes), dtype=np.float32)
|
||||
scores[0, :, 0] = 1.0
|
||||
scores[0, box_index, 0] = 1.0 - prob
|
||||
scores[0, box_index, class_id] = prob
|
||||
|
||||
return [boxes, scores]
|
||||
|
||||
def test_decodes_an_already_decoded_box(self):
|
||||
outputs = self.build_ssd_output(
|
||||
box_index=7, box=(0.1, 0.2, 0.5, 0.8), class_id=5, prob=0.9
|
||||
)
|
||||
|
||||
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
|
||||
|
||||
# boxes pass through untouched apart from the corner reorder, and
|
||||
# the background class is dropped so labels shift down by one
|
||||
self.assertEqual(detections[0][0], 4)
|
||||
self.assertAlmostEqual(float(detections[0][1]), 0.9, places=5)
|
||||
self.assertAlmostEqual(float(detections[0][2]), 0.2, places=5) # y_min
|
||||
self.assertAlmostEqual(float(detections[0][3]), 0.1, places=5) # x_min
|
||||
self.assertAlmostEqual(float(detections[0][4]), 0.8, places=5) # y_max
|
||||
self.assertAlmostEqual(float(detections[0][5]), 0.5, places=5) # x_max
|
||||
|
||||
def test_confidences_are_not_softmaxed_again(self):
|
||||
"""A second softmax over 21 already-normalized probabilities pulls
|
||||
every score toward 1/21, which is what silently dropped every real
|
||||
detection under the default threshold."""
|
||||
outputs = self.build_ssd_output(
|
||||
box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=15, prob=0.6
|
||||
)
|
||||
|
||||
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
|
||||
|
||||
self.assertAlmostEqual(float(detections[0][1]), 0.6, places=5)
|
||||
|
||||
def test_any_box_count_is_accepted(self):
|
||||
"""3000 for the MobileNet backbones, 8732 for VGG16 -- the decoder
|
||||
reads whatever the model emits rather than a per-backbone table."""
|
||||
for num_boxes in (3000, 8732):
|
||||
with self.subTest(num_boxes=num_boxes):
|
||||
outputs = self.build_ssd_output(
|
||||
box_index=num_boxes - 1,
|
||||
box=(0.0, 0.0, 1.0, 1.0),
|
||||
class_id=1,
|
||||
prob=0.8,
|
||||
num_boxes=num_boxes,
|
||||
)
|
||||
|
||||
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
|
||||
|
||||
self.assertEqual(detections[0][0], 0)
|
||||
self.assertAlmostEqual(float(detections[0][1]), 0.8, places=5)
|
||||
|
||||
def test_output_order_does_not_matter(self):
|
||||
boxes, scores = self.build_ssd_output(
|
||||
box_index=1, box=(0.2, 0.2, 0.6, 0.6), class_id=3, prob=0.7
|
||||
)
|
||||
|
||||
forward = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45)
|
||||
reversed_ = decode_ssd_raw([scores, boxes], 300, 300, 0.25, 0.45)
|
||||
|
||||
np.testing.assert_array_equal(forward, reversed_)
|
||||
|
||||
def test_low_confidence_detection_is_dropped(self):
|
||||
outputs = self.build_ssd_output(
|
||||
box_index=0, box=(0.1, 0.1, 0.4, 0.4), class_id=3, prob=0.05
|
||||
)
|
||||
|
||||
detections = decode_ssd_raw(outputs, 300, 300, 0.5, 0.45)
|
||||
|
||||
self.assertTrue(np.all(detections == 0))
|
||||
|
||||
def test_returns_empty_detections_when_outputs_cannot_be_told_apart(self):
|
||||
# two tensors that both look class-shaped (last dim != 4)
|
||||
outputs = [
|
||||
np.zeros((1, 10, 21), dtype=np.float32),
|
||||
np.zeros((1, 10, 21), dtype=np.float32),
|
||||
]
|
||||
|
||||
detections = decode_ssd_raw(outputs, 300, 300, 0.25, 0.45)
|
||||
|
||||
self.assertTrue(np.all(detections == 0))
|
||||
|
||||
def test_returns_empty_detections_when_box_and_class_counts_disagree(self):
|
||||
boxes = np.zeros((1, 10, 4), dtype=np.float32)
|
||||
scores = np.zeros((1, 12, 21), dtype=np.float32)
|
||||
|
||||
detections = decode_ssd_raw([boxes, scores], 300, 300, 0.25, 0.45)
|
||||
|
||||
self.assertTrue(np.all(detections == 0))
|
||||
|
||||
|
||||
class TestDeepxDamoyoloDecode(unittest.TestCase):
|
||||
def total_priors(self, width, height, strides):
|
||||
return sum((height // s) * (width // s) for s in strides)
|
||||
@ -743,15 +782,86 @@ if __name__ == "__main__":
|
||||
|
||||
class TestDeepxModelType(unittest.TestCase):
|
||||
def test_model_type_defaults_to_yolo_generic(self):
|
||||
"""Frigate's own model_type default is ssd, which silently picked the
|
||||
SSD decoder for anyone who left it unset."""
|
||||
"""Frigate's own model_type default is ssd, which this detector has no
|
||||
decoder for, so the detector needs a default of its own."""
|
||||
config = DeepxDetectorConfig(type="deepx")
|
||||
|
||||
self.assertIs(config.model_type, DeepxModelTypeEnum.yologeneric)
|
||||
|
||||
def test_ssd_is_not_an_accepted_model_type(self):
|
||||
"""The ModelZoo's SSD models are Pascal VOC, whose label names none of
|
||||
Frigate's COCO-based object config would ever match, so the type is
|
||||
rejected at config load rather than decoded into a wrong label map."""
|
||||
with self.assertRaises(ValidationError):
|
||||
DeepxDetectorConfig(type="deepx", model_type="ssd")
|
||||
|
||||
def test_model_type_values_match_frigate_model_types(self):
|
||||
"""The detector writes its choice back onto model.model_type, so every
|
||||
value has to round-trip through ModelTypeEnum."""
|
||||
for value in DeepxModelTypeEnum:
|
||||
with self.subTest(value=value):
|
||||
self.assertEqual(ModelTypeEnum(value.value).value, value.value)
|
||||
|
||||
|
||||
class TestDeepxIpcEndpoint(unittest.TestCase):
|
||||
"""dxrtd listens on an abstract and a filesystem socket; only the second
|
||||
one is reachable from a container, so the detector names it up front."""
|
||||
|
||||
def _construct(self):
|
||||
# dx_engine is not installed in the test environment, and forcing the
|
||||
# import to fail keeps this test honest on a machine where it is
|
||||
with patch.dict(sys.modules, {"dx_engine": None}):
|
||||
with self.assertRaises(ImportError):
|
||||
DeepxDetector(DeepxDetectorConfig(type="deepx"))
|
||||
|
||||
def test_the_filesystem_socket_is_named_when_nothing_else_is(self):
|
||||
with patch.dict(os.environ):
|
||||
os.environ.pop(DXRT_IPC_ENDPOINT_ENV, None)
|
||||
self._construct()
|
||||
self.assertEqual(os.environ[DXRT_IPC_ENDPOINT_ENV], DXRT_IPC_SOCKET)
|
||||
|
||||
def test_an_operator_supplied_endpoint_is_left_alone(self):
|
||||
with patch.dict(os.environ, {DXRT_IPC_ENDPOINT_ENV: "/run/dxrt/custom.sock"}):
|
||||
self._construct()
|
||||
self.assertEqual(os.environ[DXRT_IPC_ENDPOINT_ENV], "/run/dxrt/custom.sock")
|
||||
|
||||
|
||||
class TestDeepxServiceCheck(unittest.TestCase):
|
||||
"""DX-RT scans /proc for a dxrtd process to decide the daemon is up, which
|
||||
a container cannot satisfy from the host. See satisfy_service_check."""
|
||||
|
||||
def setUp(self):
|
||||
deepx = sys.modules["frigate.detectors.plugins.deepx"]
|
||||
self.deepx = deepx
|
||||
deepx._SERVICE_PLACEHOLDER = None
|
||||
self.addCleanup(setattr, deepx, "_SERVICE_PLACEHOLDER", None)
|
||||
|
||||
def test_nothing_is_started_when_a_daemon_is_already_visible(self):
|
||||
with patch.object(self.deepx, "service_is_visible", return_value=True):
|
||||
with patch.object(self.deepx.subprocess, "Popen") as popen:
|
||||
satisfy_service_check("/tmp/dxrt_dynamic_ipc.sock")
|
||||
|
||||
popen.assert_not_called()
|
||||
|
||||
def test_nothing_is_started_when_the_socket_is_absent(self):
|
||||
# a genuinely stopped daemon must still report itself
|
||||
with patch.object(self.deepx, "service_is_visible", return_value=False):
|
||||
with patch.object(self.deepx.subprocess, "Popen") as popen:
|
||||
satisfy_service_check("/nonexistent/dxrt.sock")
|
||||
|
||||
popen.assert_not_called()
|
||||
|
||||
def test_a_placeholder_named_dxrtd_is_started_otherwise(self):
|
||||
with patch.object(self.deepx, "service_is_visible", return_value=False):
|
||||
with patch.object(self.deepx.os.path, "exists", return_value=True):
|
||||
with patch.object(self.deepx.subprocess, "Popen") as popen:
|
||||
popen.return_value.poll.return_value = None
|
||||
satisfy_service_check("/tmp/dxrt_dynamic_ipc.sock")
|
||||
|
||||
args, kwargs = popen.call_args
|
||||
self.assertEqual(args[0][0], "dxrtd")
|
||||
self.assertEqual(kwargs["executable"], "/bin/sleep")
|
||||
|
||||
def test_the_scan_matches_the_runtime_own_check(self):
|
||||
# our own cmdline does not name dxrtd, so a bare test run sees none
|
||||
self.assertIsInstance(service_is_visible(), bool)
|
||||
|
||||
@ -184,6 +184,26 @@ class TestAccelerators(HardwareProbeTestCase):
|
||||
)
|
||||
self.assertFalse(memryx.unlimited)
|
||||
|
||||
def test_each_deepx_node_is_a_unit(self):
|
||||
write(os.path.join(self.dev_root, "dxrt0"))
|
||||
write(os.path.join(self.dev_root, "dxrt1"))
|
||||
|
||||
deepx = self.probe()["deepx"]
|
||||
|
||||
self.assertEqual(
|
||||
[unit.device for unit in deepx.units],
|
||||
["deepx:PCIe:0", "deepx:PCIe:1"],
|
||||
)
|
||||
|
||||
def test_a_deepx_npu_is_unlimited(self):
|
||||
# the host daemon multiplexes, so one module takes several processes
|
||||
write(os.path.join(self.dev_root, "dxrt0"))
|
||||
|
||||
self.assertTrue(self.probe()["deepx"].unlimited)
|
||||
|
||||
def test_no_deepx_is_reported_without_a_node(self):
|
||||
self.assertNotIn("deepx", self.probe())
|
||||
|
||||
def test_a_supported_rockchip_soc_is_reported(self):
|
||||
write(
|
||||
os.path.join(self.proc_root, "device-tree", "compatible"),
|
||||
|
||||
@ -324,7 +324,7 @@
|
||||
},
|
||||
"model_type": {
|
||||
"label": "Object Detection Model Type",
|
||||
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine) used by some detectors for optimization."
|
||||
"description": "Detector model architecture type (ssd, yolox, yolonas, yolo-generic, rfdetr, dfine, damo-yolo) used by some detectors for optimization."
|
||||
}
|
||||
},
|
||||
"genai": {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user