mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-05-03 06:50:58 +00:00
Compare commits
No commits in common. "e6cbc93703e208416e448a807494a96dc4e4a4d7" and "90db2d57b3fcecc302b8976fd2d7a29d3ed2836c" have entirely different histories.
e6cbc93703
...
90db2d57b3
25
.github/workflows/ci.yml
vendored
25
.github/workflows/ci.yml
vendored
@ -173,31 +173,6 @@ jobs:
|
||||
set: |
|
||||
rk.tags=${{ steps.setup.outputs.image-name }}-rk
|
||||
*.cache-from=type=gha
|
||||
synaptics_build:
|
||||
runs-on: ubuntu-22.04-arm
|
||||
name: Synaptics Build
|
||||
needs:
|
||||
- arm64_build
|
||||
steps:
|
||||
- name: Check out code
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up QEMU and Buildx
|
||||
id: setup
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Build and push Synaptics build
|
||||
uses: docker/bake-action@v6
|
||||
with:
|
||||
source: .
|
||||
push: true
|
||||
targets: synaptics
|
||||
files: docker/synaptics/synaptics.hcl
|
||||
set: |
|
||||
synaptics.tags=${{ steps.setup.outputs.image-name }}-synaptics
|
||||
*.cache-from=type=gha
|
||||
# The majority of users running arm64 are rpi users, so the rpi
|
||||
# build should be the primary arm64 image
|
||||
assemble_default_build:
|
||||
|
||||
55
.github/workflows/pull_request.yml
vendored
55
.github/workflows/pull_request.yml
vendored
@ -4,14 +4,38 @@ on:
|
||||
pull_request:
|
||||
paths-ignore:
|
||||
- "docs/**"
|
||||
- ".github/*.yml"
|
||||
- ".github/DISCUSSION_TEMPLATE/**"
|
||||
- ".github/ISSUE_TEMPLATE/**"
|
||||
- ".github/**"
|
||||
|
||||
env:
|
||||
DEFAULT_PYTHON: 3.11
|
||||
|
||||
jobs:
|
||||
build_devcontainer:
|
||||
runs-on: ubuntu-latest
|
||||
name: Build Devcontainer
|
||||
# The Dockerfile contains features that requires buildkit, and since the
|
||||
# devcontainer cli uses docker-compose to build the image, the only way to
|
||||
# ensure docker-compose uses buildkit is to explicitly enable it.
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@master
|
||||
with:
|
||||
node-version: 20.x
|
||||
- name: Install devcontainer cli
|
||||
run: npm install --global @devcontainers/cli
|
||||
- name: Build devcontainer
|
||||
run: devcontainer build --workspace-folder .
|
||||
# It would be nice to also test the following commands, but for some
|
||||
# reason they don't work even though in VS Code devcontainer works.
|
||||
# - name: Start devcontainer
|
||||
# run: devcontainer up --workspace-folder .
|
||||
# - name: Run devcontainer scripts
|
||||
# run: devcontainer run-user-commands --workspace-folder .
|
||||
|
||||
web_lint:
|
||||
name: Web - Lint
|
||||
runs-on: ubuntu-latest
|
||||
@ -78,18 +102,13 @@ jobs:
|
||||
uses: actions/checkout@v5
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@master
|
||||
with:
|
||||
node-version: 20.x
|
||||
- name: Install devcontainer cli
|
||||
run: npm install --global @devcontainers/cli
|
||||
- name: Build devcontainer
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
run: devcontainer build --workspace-folder .
|
||||
- name: Start devcontainer
|
||||
run: devcontainer up --workspace-folder .
|
||||
- name: Run mypy in devcontainer
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m mypy --config-file frigate/mypy.ini frigate"
|
||||
- name: Run unit tests in devcontainer
|
||||
run: devcontainer exec --workspace-folder . bash -lc "python3 -u -m unittest"
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
- name: Build
|
||||
run: make debug
|
||||
- name: Run mypy
|
||||
run: docker run --rm --entrypoint=python3 frigate:latest -u -m mypy --config-file frigate/mypy.ini frigate
|
||||
- name: Run tests
|
||||
run: docker run --rm --entrypoint=python3 frigate:latest -u -m unittest
|
||||
|
||||
@ -55,7 +55,7 @@ RUN --mount=type=tmpfs,target=/tmp --mount=type=tmpfs,target=/var/cache/apt \
|
||||
FROM scratch AS go2rtc
|
||||
ARG TARGETARCH
|
||||
WORKDIR /rootfs/usr/local/go2rtc/bin
|
||||
ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.9.10/go2rtc_linux_${TARGETARCH}" go2rtc
|
||||
ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.9.9/go2rtc_linux_${TARGETARCH}" go2rtc
|
||||
|
||||
FROM wget AS tempio
|
||||
ARG TARGETARCH
|
||||
|
||||
@ -1,28 +0,0 @@
|
||||
# syntax=docker/dockerfile:1.6
|
||||
|
||||
# https://askubuntu.com/questions/972516/debian-frontend-environment-variable
|
||||
ARG DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Globally set pip break-system-packages option to avoid having to specify it every time
|
||||
ARG PIP_BREAK_SYSTEM_PACKAGES=1
|
||||
|
||||
FROM wheels AS synap1680-wheels
|
||||
ARG TARGETARCH
|
||||
|
||||
# Install dependencies
|
||||
RUN wget -qO- "https://github.com/GaryHuang-ASUS/synaptics_astra_sdk/releases/download/v1.5.0/Synaptics-SL1680-v1.5.0-rt.tar" | tar -C / -xzf -
|
||||
RUN wget -P /wheels/ "https://github.com/synaptics-synap/synap-python/releases/download/v0.0.4-preview/synap_python-0.0.4-cp311-cp311-manylinux_2_35_aarch64.whl"
|
||||
|
||||
FROM deps AS synap1680-deps
|
||||
ARG TARGETARCH
|
||||
ARG PIP_BREAK_SYSTEM_PACKAGES
|
||||
|
||||
RUN --mount=type=bind,from=synap1680-wheels,source=/wheels,target=/deps/synap-wheels \
|
||||
pip3 install --no-deps -U /deps/synap-wheels/*.whl
|
||||
|
||||
WORKDIR /opt/frigate/
|
||||
COPY --from=rootfs / /
|
||||
|
||||
COPY --from=synap1680-wheels /rootfs/usr/local/lib/*.so /usr/lib
|
||||
|
||||
ADD https://raw.githubusercontent.com/synaptics-astra/synap-release/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80/model.synap /synaptics/mobilenet.synap
|
||||
@ -1,27 +0,0 @@
|
||||
target wheels {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "wheels"
|
||||
}
|
||||
|
||||
target deps {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "deps"
|
||||
}
|
||||
|
||||
target rootfs {
|
||||
dockerfile = "docker/main/Dockerfile"
|
||||
platforms = ["linux/arm64"]
|
||||
target = "rootfs"
|
||||
}
|
||||
|
||||
target synaptics {
|
||||
dockerfile = "docker/synaptics/Dockerfile"
|
||||
contexts = {
|
||||
wheels = "target:wheels",
|
||||
deps = "target:deps",
|
||||
rootfs = "target:rootfs"
|
||||
}
|
||||
platforms = ["linux/arm64"]
|
||||
}
|
||||
@ -1,15 +0,0 @@
|
||||
BOARDS += synaptics
|
||||
|
||||
local-synaptics: version
|
||||
docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
|
||||
--set synaptics.tags=frigate:latest-synaptics \
|
||||
--load
|
||||
|
||||
build-synaptics: version
|
||||
docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
|
||||
--set synaptics.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-synaptics
|
||||
|
||||
push-synaptics: build-synaptics
|
||||
docker buildx bake --file=docker/synaptics/synaptics.hcl synaptics \
|
||||
--set synaptics.tags=$(IMAGE_REPO):${GITHUB_REF_NAME}-$(COMMIT_HASH)-synaptics \
|
||||
--push
|
||||
@ -177,11 +177,9 @@ listen [::]:5000 ipv6only=off;
|
||||
By default, Frigate runs at the root path (`/`). However some setups require to run Frigate under a custom path prefix (e.g. `/frigate`), especially when Frigate is located behind a reverse proxy that requires path-based routing.
|
||||
|
||||
### Set Base Path via HTTP Header
|
||||
|
||||
The preferred way to configure the base path is through the `X-Ingress-Path` HTTP header, which needs to be set to the desired base path in an upstream reverse proxy.
|
||||
|
||||
For example, in Nginx:
|
||||
|
||||
```
|
||||
location /frigate {
|
||||
proxy_set_header X-Ingress-Path /frigate;
|
||||
@ -190,11 +188,9 @@ location /frigate {
|
||||
```
|
||||
|
||||
### Set Base Path via Environment Variable
|
||||
|
||||
When it is not feasible to set the base path via a HTTP header, it can also be set via the `FRIGATE_BASE_PATH` environment variable in the Docker Compose file.
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
services:
|
||||
frigate:
|
||||
@ -204,7 +200,6 @@ services:
|
||||
```
|
||||
|
||||
This can be used for example to access Frigate via a Tailscale agent (https), by simply forwarding all requests to the base path (http):
|
||||
|
||||
```
|
||||
tailscale serve --https=443 --bg --set-path /frigate http://localhost:5000/frigate
|
||||
```
|
||||
@ -223,7 +218,7 @@ To do this:
|
||||
|
||||
### Custom go2rtc version
|
||||
|
||||
Frigate currently includes go2rtc v1.9.10, there may be certain cases where you want to run a different version of go2rtc.
|
||||
Frigate currently includes go2rtc v1.9.9, there may be certain cases where you want to run a different version of go2rtc.
|
||||
|
||||
To do this:
|
||||
|
||||
|
||||
@ -231,7 +231,7 @@ go2rtc:
|
||||
- rtspx://192.168.1.1:7441/abcdefghijk
|
||||
```
|
||||
|
||||
[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#source-rtsp)
|
||||
[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#source-rtsp)
|
||||
|
||||
In the Unifi 2.0 update Unifi Protect Cameras had a change in audio sample rate which causes issues for ffmpeg. The input rate needs to be set for record if used directly with unifi protect.
|
||||
|
||||
|
||||
@ -427,29 +427,3 @@ cameras:
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
## Synaptics
|
||||
|
||||
Hardware accelerated video de-/encoding is supported on Synpatics SL-series SoC.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Make sure to follow the [Synaptics specific installation instructions](/frigate/installation#synaptics).
|
||||
|
||||
### Configuration
|
||||
|
||||
Add one of the following FFmpeg presets to your `config.yml` to enable hardware video processing:
|
||||
|
||||
```yaml
|
||||
ffmpeg:
|
||||
hwaccel_args: -c:v h264_v4l2m2m
|
||||
input_args: preset-rtsp-restream
|
||||
output_args:
|
||||
record: preset-record-generic-audio-aac
|
||||
```
|
||||
|
||||
:::warning
|
||||
|
||||
Make sure that your SoC supports hardware acceleration for your input stream and your input stream is h264 encoding. For example, if your camera streams with h264 encoding, your SoC must be able to de- and encode with it. If you are unsure whether your SoC meets the requirements, take a look at the datasheet.
|
||||
|
||||
:::
|
||||
|
||||
@ -43,10 +43,6 @@ Frigate supports multiple different detectors that work on different types of ha
|
||||
|
||||
- [RKNN](#rockchip-platform): RKNN models can run on Rockchip devices with included NPUs.
|
||||
|
||||
**Synaptics**
|
||||
|
||||
- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs.
|
||||
|
||||
**For Testing**
|
||||
|
||||
- [CPU Detector (not recommended for actual use](#cpu-detector-not-recommended): Use a CPU to run tflite model, this is not recommended and in most cases OpenVINO can be used in CPU mode with better results.
|
||||
@ -453,13 +449,12 @@ The YOLO detector has been designed to support YOLOv3, YOLOv4, YOLOv7, and YOLOv
|
||||
|
||||
:::
|
||||
|
||||
When Frigate is started with the following config it will connect to the detector client and transfer the model automatically:
|
||||
After placing the downloaded onnx model in your config folder, you can use the following configuration:
|
||||
|
||||
```yaml
|
||||
detectors:
|
||||
apple-silicon:
|
||||
type: zmq
|
||||
endpoint: tcp://host.docker.internal:5555
|
||||
onnx:
|
||||
type: onnx
|
||||
|
||||
model:
|
||||
model_type: yolo-generic
|
||||
@ -1053,41 +1048,6 @@ model:
|
||||
height: 320 # MUST match the chosen model i.e yolov7-320 -> 320 yolov4-416 -> 416
|
||||
```
|
||||
|
||||
## Synaptics
|
||||
|
||||
Hardware accelerated object detection is supported on the following SoCs:
|
||||
|
||||
- SL1680
|
||||
|
||||
This implementation uses the [Synaptics model conversion](https://synaptics-synap.github.io/doc/v/latest/docs/manual/introduction.html#offline-model-conversion), version v3.1.0.
|
||||
|
||||
This implementation is based on sdk `v1.5.0`.
|
||||
|
||||
See the [installation docs](../frigate/installation.md#synaptics) for information on configuring the SL-series NPU hardware.
|
||||
|
||||
### Configuration
|
||||
|
||||
When configuring the Synap detector, you have to specify the model: a local **path**.
|
||||
|
||||
#### SSD Mobilenet
|
||||
|
||||
A synap model is provided in the container at /mobilenet.synap and is used by this detector type by default. The model comes from [Synap-release Github](https://github.com/synaptics-astra/synap-release/tree/v1.5.0/models/dolphin/object_detection/coco/model/mobilenet224_full80).
|
||||
|
||||
Use the model configuration shown below when using the synaptics detector with the default synap model:
|
||||
|
||||
```yaml
|
||||
detectors: # required
|
||||
synap_npu: # required
|
||||
type: synaptics # required
|
||||
|
||||
model: # required
|
||||
path: /synaptics/mobilenet.synap # required
|
||||
width: 224 # required
|
||||
height: 224 # required
|
||||
tensor_format: nhwc # default value (optional. If you change the model, it is required)
|
||||
labelmap_path: /labelmap/coco-80.txt # required
|
||||
```
|
||||
|
||||
## Rockchip platform
|
||||
|
||||
Hardware accelerated object detection is supported on the following SoCs:
|
||||
|
||||
@ -287,9 +287,6 @@ detect:
|
||||
max_disappeared: 25
|
||||
# Optional: Configuration for stationary object tracking
|
||||
stationary:
|
||||
# Optional: Stationary classifier that uses visual characteristics to determine if an object
|
||||
# is stationary even if the box changes enough to be considered motion (default: shown below).
|
||||
classifier: True
|
||||
# Optional: Frequency for confirming stationary objects (default: same as threshold)
|
||||
# When set to 1, object detection will run to confirm the object still exists on every frame.
|
||||
# If set to 10, object detection will run to confirm the object still exists on every 10th frame.
|
||||
@ -700,7 +697,7 @@ audio_transcription:
|
||||
language: en
|
||||
|
||||
# Optional: Restream configuration
|
||||
# Uses https://github.com/AlexxIT/go2rtc (v1.9.10)
|
||||
# Uses https://github.com/AlexxIT/go2rtc (v1.9.9)
|
||||
# NOTE: The default go2rtc API port (1984) must be used,
|
||||
# changing this port for the integrated go2rtc instance is not supported.
|
||||
go2rtc:
|
||||
|
||||
@ -7,7 +7,7 @@ title: Restream
|
||||
|
||||
Frigate can restream your video feed as an RTSP feed for other applications such as Home Assistant to utilize it at `rtsp://<frigate_host>:8554/<camera_name>`. Port 8554 must be open. [This allows you to use a video feed for detection in Frigate and Home Assistant live view at the same time without having to make two separate connections to the camera](#reduce-connections-to-camera). The video feed is copied from the original video feed directly to avoid re-encoding. This feed does not include any annotation by Frigate.
|
||||
|
||||
Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.9.10) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#configuration) for more advanced configurations and features.
|
||||
Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.9.9) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#configuration) for more advanced configurations and features.
|
||||
|
||||
:::note
|
||||
|
||||
@ -156,7 +156,7 @@ See [this comment](https://github.com/AlexxIT/go2rtc/issues/1217#issuecomment-22
|
||||
|
||||
## Advanced Restream Configurations
|
||||
|
||||
The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below:
|
||||
The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below:
|
||||
|
||||
NOTE: The output will need to be passed with two curly braces `{{output}}`
|
||||
|
||||
|
||||
@ -95,21 +95,8 @@ Frigate supports multiple different detectors that work on different types of ha
|
||||
- Runs best with tiny or small size models
|
||||
- Runs efficiently on low power hardware
|
||||
|
||||
**Synaptics**
|
||||
|
||||
- [Synaptics](#synaptics): synap models can run on Synaptics devices(e.g astra machina) with included NPUs to provide efficient object detection.
|
||||
|
||||
:::
|
||||
|
||||
### Synaptics
|
||||
|
||||
- **Synaptics** Default model is **mobilenet**
|
||||
|
||||
| Name | Synaptics SL1680 Inference Time |
|
||||
| ---------------- | ------------------------------- |
|
||||
| ssd mobilenet | ~ 25 ms |
|
||||
| yolov5m | ~ 118 ms |
|
||||
|
||||
### Hailo-8
|
||||
|
||||
Frigate supports both the Hailo-8 and Hailo-8L AI Acceleration Modules on compatible hardware platforms—including the Raspberry Pi 5 with the PCIe hat from the AI kit. The Hailo detector integration in Frigate automatically identifies your hardware type and selects the appropriate default model when a custom model isn’t provided.
|
||||
|
||||
@ -256,37 +256,6 @@ or add these options to your `docker run` command:
|
||||
|
||||
Next, you should configure [hardware object detection](/configuration/object_detectors#rockchip-platform) and [hardware video processing](/configuration/hardware_acceleration_video#rockchip-platform).
|
||||
|
||||
### Synaptics
|
||||
|
||||
- SL1680
|
||||
|
||||
#### Setup
|
||||
|
||||
Follow Frigate's default installation instructions, but use a docker image with `-synaptics` suffix for example `ghcr.io/blakeblackshear/frigate:stable-synaptics`.
|
||||
|
||||
Next, you need to grant docker permissions to access your hardware:
|
||||
|
||||
- During the configuration process, you should run docker in privileged mode to avoid any errors due to insufficient permissions. To do so, add `privileged: true` to your `docker-compose.yml` file or the `--privileged` flag to your docker run command.
|
||||
|
||||
```yaml
|
||||
devices:
|
||||
- /dev/synap
|
||||
- /dev/video0
|
||||
- /dev/video1
|
||||
```
|
||||
|
||||
or add these options to your `docker run` command:
|
||||
|
||||
```
|
||||
--device /dev/synap \
|
||||
--device /dev/video0 \
|
||||
--device /dev/video1
|
||||
```
|
||||
|
||||
#### Configuration
|
||||
|
||||
Next, you should configure [hardware object detection](/configuration/object_detectors#synaptics) and [hardware video processing](/configuration/hardware_acceleration_video#synaptics).
|
||||
|
||||
## Docker
|
||||
|
||||
Running through Docker with Docker Compose is the recommended install method.
|
||||
|
||||
@ -13,7 +13,7 @@ Use of the bundled go2rtc is optional. You can still configure FFmpeg to connect
|
||||
|
||||
# Setup a go2rtc stream
|
||||
|
||||
First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#module-streams), not just rtsp.
|
||||
First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#module-streams), not just rtsp.
|
||||
|
||||
:::tip
|
||||
|
||||
@ -49,8 +49,8 @@ After adding this to the config, restart Frigate and try to watch the live strea
|
||||
- Check Video Codec:
|
||||
|
||||
- If the camera stream works in go2rtc but not in your browser, the video codec might be unsupported.
|
||||
- If using H265, switch to H264. Refer to [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#codecs-madness) in go2rtc documentation.
|
||||
- If unable to switch from H265 to H264, or if the stream format is different (e.g., MJPEG), re-encode the video using [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.9.10#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view.
|
||||
- If using H265, switch to H264. Refer to [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#codecs-madness) in go2rtc documentation.
|
||||
- If unable to switch from H265 to H264, or if the stream format is different (e.g., MJPEG), re-encode the video using [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.9.9#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view.
|
||||
```yaml
|
||||
go2rtc:
|
||||
streams:
|
||||
|
||||
@ -5,14 +5,14 @@ import frigateHttpApiSidebar from "./docs/integrations/api/sidebar";
|
||||
const sidebars: SidebarsConfig = {
|
||||
docs: {
|
||||
Frigate: [
|
||||
"frigate/index",
|
||||
"frigate/hardware",
|
||||
"frigate/planning_setup",
|
||||
"frigate/installation",
|
||||
"frigate/updating",
|
||||
"frigate/camera_setup",
|
||||
"frigate/video_pipeline",
|
||||
"frigate/glossary",
|
||||
'frigate/index',
|
||||
'frigate/hardware',
|
||||
'frigate/planning_setup',
|
||||
'frigate/installation',
|
||||
'frigate/updating',
|
||||
'frigate/camera_setup',
|
||||
'frigate/video_pipeline',
|
||||
'frigate/glossary',
|
||||
],
|
||||
Guides: [
|
||||
"guides/getting_started",
|
||||
@ -28,7 +28,7 @@ const sidebars: SidebarsConfig = {
|
||||
{
|
||||
type: "link",
|
||||
label: "Go2RTC Configuration Reference",
|
||||
href: "https://github.com/AlexxIT/go2rtc/tree/v1.9.10#configuration",
|
||||
href: "https://github.com/AlexxIT/go2rtc/tree/v1.9.9#configuration",
|
||||
} as PropSidebarItemLink,
|
||||
],
|
||||
Detectors: [
|
||||
@ -119,11 +119,11 @@ const sidebars: SidebarsConfig = {
|
||||
"configuration/metrics",
|
||||
"integrations/third_party_extensions",
|
||||
],
|
||||
"Frigate+": [
|
||||
"plus/index",
|
||||
"plus/annotating",
|
||||
"plus/first_model",
|
||||
"plus/faq",
|
||||
'Frigate+': [
|
||||
'plus/index',
|
||||
'plus/annotating',
|
||||
'plus/first_model',
|
||||
'plus/faq',
|
||||
],
|
||||
Troubleshooting: [
|
||||
"troubleshooting/faqs",
|
||||
|
||||
@ -29,10 +29,6 @@ class StationaryConfig(FrigateBaseModel):
|
||||
default_factory=StationaryMaxFramesConfig,
|
||||
title="Max frames for stationary objects.",
|
||||
)
|
||||
classifier: bool = Field(
|
||||
default=True,
|
||||
title="Enable visual classifier for determing if objects with jittery bounding boxes are stationary.",
|
||||
)
|
||||
|
||||
|
||||
class DetectConfig(FrigateBaseModel):
|
||||
|
||||
@ -93,7 +93,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
|
||||
if camera_config.review.genai.debug_save_thumbnails:
|
||||
id = data["after"]["id"]
|
||||
Path(os.path.join(CLIPS_DIR, "genai-requests", f"{id}")).mkdir(
|
||||
Path(os.path.join(CLIPS_DIR, f"genai-requests/{id}")).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
shutil.copy(
|
||||
@ -124,9 +124,6 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
if topic == EmbeddingsRequestEnum.summarize_review.value:
|
||||
start_ts = request_data["start_ts"]
|
||||
end_ts = request_data["end_ts"]
|
||||
logger.debug(
|
||||
f"Found GenAI Review Summary request for {start_ts} to {end_ts}"
|
||||
)
|
||||
items: list[dict[str, Any]] = [
|
||||
r["data"]["metadata"]
|
||||
for r in (
|
||||
@ -144,7 +141,7 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
|
||||
if len(items) == 0:
|
||||
logger.debug("No review items with metadata found during time period")
|
||||
return "No activity was found during this time."
|
||||
return None
|
||||
|
||||
important_items = list(
|
||||
filter(
|
||||
@ -157,16 +154,8 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
if not important_items:
|
||||
return "No concerns were found during this time period."
|
||||
|
||||
if self.config.review.genai.debug_save_thumbnails:
|
||||
Path(
|
||||
os.path.join(CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}")
|
||||
).mkdir(parents=True, exist_ok=True)
|
||||
|
||||
return self.genai_client.generate_review_summary(
|
||||
start_ts,
|
||||
end_ts,
|
||||
important_items,
|
||||
self.config.review.genai.debug_save_thumbnails,
|
||||
start_ts, end_ts, important_items
|
||||
)
|
||||
else:
|
||||
return None
|
||||
|
||||
@ -19,4 +19,3 @@ class ReviewMetadata(BaseModel):
|
||||
default=None,
|
||||
description="Other concerns highlighted by the user that are observed.",
|
||||
)
|
||||
time: str | None = Field(default=None, description="Time of activity.")
|
||||
|
||||
@ -1,91 +0,0 @@
|
||||
import logging
|
||||
import os
|
||||
|
||||
import numpy as np
|
||||
from synap import Network
|
||||
from synap.postprocessor import Detector
|
||||
from synap.preprocessor import Preprocessor
|
||||
from synap.types import Layout, Shape
|
||||
from typing_extensions import Literal
|
||||
|
||||
from frigate.detectors.detection_api import DetectionApi
|
||||
from frigate.detectors.detector_config import (
|
||||
BaseDetectorConfig,
|
||||
InputTensorEnum,
|
||||
ModelTypeEnum,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTOR_KEY = "synaptics"
|
||||
|
||||
|
||||
class SynapDetectorConfig(BaseDetectorConfig):
|
||||
type: Literal[DETECTOR_KEY]
|
||||
|
||||
|
||||
class SynapDetector(DetectionApi):
|
||||
type_key = DETECTOR_KEY
|
||||
|
||||
def __init__(self, detector_config: SynapDetectorConfig):
|
||||
try:
|
||||
_, ext = os.path.splitext(detector_config.model.path)
|
||||
if ext and ext != ".synap":
|
||||
raise ValueError("Model path config for Synap1680 is wrong.")
|
||||
|
||||
synap_network = Network(detector_config.model.path)
|
||||
logger.info(f"Synap NPU loaded model: {detector_config.model.path}")
|
||||
except ValueError as ve:
|
||||
logger.error(f"Config to Synap1680 was Failed: {ve}")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to init Synap NPU: {e}")
|
||||
raise
|
||||
|
||||
self.width = detector_config.model.width
|
||||
self.height = detector_config.model.height
|
||||
self.model_type = detector_config.model.model_type
|
||||
self.network = synap_network
|
||||
self.network_input_details = self.network.inputs[0]
|
||||
self.input_tensor_layout = detector_config.model.input_tensor
|
||||
|
||||
# Create Inference Engine
|
||||
self.preprocessor = Preprocessor()
|
||||
self.detector = Detector(score_threshold=0.4, iou_threshold=0.4)
|
||||
|
||||
def detect_raw(self, tensor_input: np.ndarray):
|
||||
# It has only been testing for pre-converted mobilenet80 .tflite -> .synap model currently
|
||||
layout = Layout.nhwc # default layout
|
||||
detections = np.zeros((20, 6), np.float32)
|
||||
|
||||
if self.input_tensor_layout == InputTensorEnum.nhwc:
|
||||
layout = Layout.nhwc
|
||||
|
||||
postprocess_data = self.preprocessor.assign(
|
||||
self.network.inputs, tensor_input, Shape(tensor_input.shape), layout
|
||||
)
|
||||
output_tensor_obj = self.network.predict()
|
||||
output = self.detector.process(output_tensor_obj, postprocess_data)
|
||||
|
||||
if self.model_type == ModelTypeEnum.ssd:
|
||||
for i, item in enumerate(output.items):
|
||||
if i == 20:
|
||||
break
|
||||
|
||||
bb = item.bounding_box
|
||||
# Convert corner coordinates to normalized [0,1] range
|
||||
x1 = bb.origin.x / self.width # Top-left X
|
||||
y1 = bb.origin.y / self.height # Top-left Y
|
||||
x2 = (bb.origin.x + bb.size.x) / self.width # Bottom-right X
|
||||
y2 = (bb.origin.y + bb.size.y) / self.height # Bottom-right Y
|
||||
detections[i] = [
|
||||
item.class_index,
|
||||
float(item.confidence),
|
||||
y1,
|
||||
x1,
|
||||
y2,
|
||||
x2,
|
||||
]
|
||||
else:
|
||||
logger.error(f"Unsupported model type: {self.model_type}")
|
||||
return detections
|
||||
@ -313,7 +313,6 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
if resp is not None:
|
||||
return resp
|
||||
|
||||
logger.error(f"No processor handled the topic {topic}")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unable to handle embeddings request {e}", exc_info=True)
|
||||
|
||||
@ -73,7 +73,7 @@ Your task is to provide a clear, security-focused description of the scene that:
|
||||
Facts come first, but identifying security risks is the primary goal.
|
||||
|
||||
When forming your description:
|
||||
- Describe the people and objects exactly as seen. Include any observable environmental changes (e.g., lighting changes triggered by activity).
|
||||
- Describe the time, people, and objects exactly as seen. Include any observable environmental changes (e.g., lighting changes triggered by activity).
|
||||
- Time of day should **increase suspicion only when paired with unusual or security-relevant behaviors**. Do not raise the threat level for common residential activities (e.g., residents walking pets, retrieving mail, gardening, playing with pets, supervising children) even at unusual hours, unless other suspicious indicators are present.
|
||||
- Focus on behaviors that are uncharacteristic of innocent activity: loitering without clear purpose, avoiding cameras, inspecting vehicles/doors, changing behavior when lights activate, scanning surroundings without an apparent benign reason.
|
||||
- **Benign context override**: If scanning or looking around is clearly part of an innocent activity (such as playing with a dog, gardening, supervising children, or watching for a pet), do not treat it as suspicious.
|
||||
@ -99,7 +99,7 @@ Sequence details:
|
||||
**IMPORTANT:**
|
||||
- Values must be plain strings, floats, or integers — no nested objects, no extra commentary.
|
||||
{get_language_prompt()}
|
||||
"""
|
||||
"""
|
||||
logger.debug(
|
||||
f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
|
||||
)
|
||||
@ -135,7 +135,6 @@ Sequence details:
|
||||
if review_data["recognized_objects"]:
|
||||
metadata.potential_threat_level = 0
|
||||
|
||||
metadata.time = review_data["start"]
|
||||
return metadata
|
||||
except Exception as e:
|
||||
# rarely LLMs can fail to follow directions on output format
|
||||
@ -147,75 +146,34 @@ Sequence details:
|
||||
return None
|
||||
|
||||
def generate_review_summary(
|
||||
self,
|
||||
start_ts: float,
|
||||
end_ts: float,
|
||||
segments: list[dict[str, Any]],
|
||||
debug_save: bool,
|
||||
self, start_ts: float, end_ts: float, segments: list[dict[str, Any]]
|
||||
) -> str | None:
|
||||
"""Generate a summary of review item descriptions over a period of time."""
|
||||
time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%B %d, %Y at %I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%B %d, %Y at %I:%M %p')}"
|
||||
time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%I:%M %p')}"
|
||||
timeline_summary_prompt = f"""
|
||||
You are a security officer.
|
||||
Time range: {time_range}.
|
||||
You are a security officer. Time range: {time_range}.
|
||||
Input: JSON list with "scene", "confidence", "potential_threat_level" (1-2), "other_concerns".
|
||||
Write a report:
|
||||
|
||||
Task: Write a concise, human-presentable security report in markdown format.
|
||||
Security Summary - {time_range}
|
||||
[One-sentence overview of activity]
|
||||
[Chronological bullet list of events with timestamps if in scene]
|
||||
[Final threat assessment]
|
||||
|
||||
Rules for the report:
|
||||
|
||||
- Title & overview
|
||||
- Start with:
|
||||
# Security Summary - {time_range}
|
||||
- Write a 1-2 sentence situational overview capturing the general pattern of the period.
|
||||
|
||||
- Event details
|
||||
- Present events in chronological order as a bullet list.
|
||||
- **If multiple events occur within the same minute or overlapping time range, COMBINE them into a single bullet.**
|
||||
- Summarize the distinct activities as sub-points under the shared timestamp.
|
||||
- If no timestamp is given, preserve order but label as “Time not specified.”
|
||||
- Use bold timestamps for clarity.
|
||||
- Group bullets under subheadings when multiple events fall into the same category (e.g., Vehicle Activity, Porch Activity, Unusual Behavior).
|
||||
|
||||
- Threat levels
|
||||
- Always show (threat level: X) for each event.
|
||||
- If multiple events at the same time share the same threat level, only state it once.
|
||||
|
||||
- Final assessment
|
||||
- End with a Final Assessment section.
|
||||
- If all events are threat level 1 with no escalation:
|
||||
Final assessment: Only normal residential activity observed during this period.
|
||||
- If threat level 2+ events are present, clearly summarize them as Potential concerns requiring review.
|
||||
|
||||
- Conciseness
|
||||
- Do not repeat benign clothing/appearance details unless they distinguish individuals.
|
||||
- Summarize similar routine events instead of restating full scene descriptions.
|
||||
"""
|
||||
Rules:
|
||||
- List events in order.
|
||||
- Highlight potential_threat_level ≥ 1 with exact times.
|
||||
- Note any of the additional concerns which are present.
|
||||
- Note unusual activity even if not threats.
|
||||
- If no threats: "Final assessment: Only normal activity observed during this period."
|
||||
- No commentary, questions, or recommendations.
|
||||
- Output only the report.
|
||||
"""
|
||||
|
||||
for item in segments:
|
||||
timeline_summary_prompt += f"\n{item}"
|
||||
|
||||
if debug_save:
|
||||
with open(
|
||||
os.path.join(
|
||||
CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}", "prompt.txt"
|
||||
),
|
||||
"w",
|
||||
) as f:
|
||||
f.write(timeline_summary_prompt)
|
||||
|
||||
response = self._send(timeline_summary_prompt, [])
|
||||
|
||||
if debug_save and response:
|
||||
with open(
|
||||
os.path.join(
|
||||
CLIPS_DIR, "genai-requests", f"{start_ts}-{end_ts}", "response.txt"
|
||||
),
|
||||
"w",
|
||||
) as f:
|
||||
f.write(response)
|
||||
|
||||
return response
|
||||
return self._send(timeline_summary_prompt, [])
|
||||
|
||||
def generate_object_description(
|
||||
self,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
import logging
|
||||
import random
|
||||
import string
|
||||
from typing import Any, Sequence, cast
|
||||
from typing import Any, Sequence
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
@ -17,7 +17,6 @@ from frigate.camera import PTZMetrics
|
||||
from frigate.config import CameraConfig
|
||||
from frigate.ptz.autotrack import PtzMotionEstimator
|
||||
from frigate.track import ObjectTracker
|
||||
from frigate.track.stationary_classifier import StationaryMotionClassifier
|
||||
from frigate.util.image import (
|
||||
SharedMemoryFrameManager,
|
||||
get_histogram,
|
||||
@ -120,7 +119,6 @@ class NorfairTracker(ObjectTracker):
|
||||
self.ptz_motion_estimator: PtzMotionEstimator | None = None
|
||||
self.camera_name = config.name
|
||||
self.track_id_map: dict[str, str] = {}
|
||||
self.stationary_classifier = StationaryMotionClassifier()
|
||||
|
||||
# Define tracker configurations for static camera
|
||||
self.object_type_configs = {
|
||||
@ -323,14 +321,23 @@ class NorfairTracker(ObjectTracker):
|
||||
|
||||
# tracks the current position of the object based on the last N bounding boxes
|
||||
# returns False if the object has moved outside its previous position
|
||||
def update_position(
|
||||
self,
|
||||
id: str,
|
||||
box: list[int],
|
||||
stationary: bool,
|
||||
yuv_frame: np.ndarray | None,
|
||||
) -> bool:
|
||||
def reset_position(xmin: int, ymin: int, xmax: int, ymax: int) -> None:
|
||||
def update_position(self, id: str, box: list[int], stationary: bool) -> bool:
|
||||
xmin, ymin, xmax, ymax = box
|
||||
position = self.positions[id]
|
||||
self.stationary_box_history[id].append(box)
|
||||
|
||||
if len(self.stationary_box_history[id]) > MAX_STATIONARY_HISTORY:
|
||||
self.stationary_box_history[id] = self.stationary_box_history[id][
|
||||
-MAX_STATIONARY_HISTORY:
|
||||
]
|
||||
|
||||
avg_iou = intersection_over_union(
|
||||
box, average_boxes(self.stationary_box_history[id])
|
||||
)
|
||||
|
||||
# object has minimal or zero iou
|
||||
# assume object is active
|
||||
if avg_iou < THRESHOLD_KNOWN_ACTIVE_IOU:
|
||||
self.positions[id] = {
|
||||
"xmins": [xmin],
|
||||
"ymins": [ymin],
|
||||
@ -341,48 +348,13 @@ class NorfairTracker(ObjectTracker):
|
||||
"xmax": xmax,
|
||||
"ymax": ymax,
|
||||
}
|
||||
|
||||
xmin, ymin, xmax, ymax = box
|
||||
position = self.positions[id]
|
||||
self.stationary_box_history[id].append(box)
|
||||
|
||||
if len(self.stationary_box_history[id]) > MAX_STATIONARY_HISTORY:
|
||||
self.stationary_box_history[id] = self.stationary_box_history[id][
|
||||
-MAX_STATIONARY_HISTORY:
|
||||
]
|
||||
|
||||
avg_box = average_boxes(self.stationary_box_history[id])
|
||||
avg_iou = intersection_over_union(box, avg_box)
|
||||
median_box = median_of_boxes(self.stationary_box_history[id])
|
||||
|
||||
# Establish anchor early when stationary and stable
|
||||
if stationary and yuv_frame is not None:
|
||||
history = self.stationary_box_history[id]
|
||||
if id not in self.stationary_classifier.anchor_crops and len(history) >= 5:
|
||||
stability_iou = intersection_over_union(avg_box, median_box)
|
||||
if stability_iou >= 0.7:
|
||||
self.stationary_classifier.ensure_anchor(
|
||||
id, yuv_frame, cast(tuple[int, int, int, int], median_box)
|
||||
)
|
||||
|
||||
# object has minimal or zero iou
|
||||
# assume object is active
|
||||
if avg_iou < THRESHOLD_KNOWN_ACTIVE_IOU:
|
||||
if stationary and yuv_frame is not None:
|
||||
if not self.stationary_classifier.evaluate(
|
||||
id, yuv_frame, cast(tuple[int, int, int, int], tuple(box))
|
||||
):
|
||||
reset_position(xmin, ymin, xmax, ymax)
|
||||
return False
|
||||
else:
|
||||
reset_position(xmin, ymin, xmax, ymax)
|
||||
return False
|
||||
return False
|
||||
|
||||
threshold = (
|
||||
THRESHOLD_STATIONARY_CHECK_IOU if stationary else THRESHOLD_ACTIVE_CHECK_IOU
|
||||
)
|
||||
|
||||
# object has iou below threshold, check median and optionally crop similarity
|
||||
# object has iou below threshold, check median to reduce outliers
|
||||
if avg_iou < threshold:
|
||||
median_iou = intersection_over_union(
|
||||
(
|
||||
@ -391,26 +363,27 @@ class NorfairTracker(ObjectTracker):
|
||||
position["xmax"],
|
||||
position["ymax"],
|
||||
),
|
||||
median_box,
|
||||
median_of_boxes(self.stationary_box_history[id]),
|
||||
)
|
||||
|
||||
# if the median iou drops below the threshold
|
||||
# assume object is no longer stationary
|
||||
if median_iou < threshold:
|
||||
# If we have a yuv_frame to check before flipping to active, check with classifier if we have YUV frame
|
||||
if stationary and yuv_frame is not None:
|
||||
if not self.stationary_classifier.evaluate(
|
||||
id, yuv_frame, cast(tuple[int, int, int, int], tuple(box))
|
||||
):
|
||||
reset_position(xmin, ymin, xmax, ymax)
|
||||
return False
|
||||
else:
|
||||
reset_position(xmin, ymin, xmax, ymax)
|
||||
return False
|
||||
self.positions[id] = {
|
||||
"xmins": [xmin],
|
||||
"ymins": [ymin],
|
||||
"xmaxs": [xmax],
|
||||
"ymaxs": [ymax],
|
||||
"xmin": xmin,
|
||||
"ymin": ymin,
|
||||
"xmax": xmax,
|
||||
"ymax": ymax,
|
||||
}
|
||||
return False
|
||||
|
||||
# if there are more than 5 and less than 10 entries for the position, add the bounding box
|
||||
# and recompute the position box
|
||||
if len(position["xmins"]) < 10:
|
||||
if 5 <= len(position["xmins"]) < 10:
|
||||
position["xmins"].append(xmin)
|
||||
position["ymins"].append(ymin)
|
||||
position["xmaxs"].append(xmax)
|
||||
@ -443,12 +416,7 @@ class NorfairTracker(ObjectTracker):
|
||||
|
||||
return False
|
||||
|
||||
def update(
|
||||
self,
|
||||
track_id: str,
|
||||
obj: dict[str, Any],
|
||||
yuv_frame: np.ndarray | None,
|
||||
) -> None:
|
||||
def update(self, track_id: str, obj: dict[str, Any]) -> None:
|
||||
id = self.track_id_map[track_id]
|
||||
self.disappeared[id] = 0
|
||||
stationary = (
|
||||
@ -456,7 +424,7 @@ class NorfairTracker(ObjectTracker):
|
||||
>= self.detect_config.stationary.threshold
|
||||
)
|
||||
# update the motionless count if the object has not moved to a new position
|
||||
if self.update_position(id, obj["box"], stationary, yuv_frame):
|
||||
if self.update_position(id, obj["box"], stationary):
|
||||
self.tracked_objects[id]["motionless_count"] += 1
|
||||
if self.is_expired(id):
|
||||
self.deregister(id, track_id)
|
||||
@ -472,7 +440,6 @@ class NorfairTracker(ObjectTracker):
|
||||
self.tracked_objects[id]["position_changes"] += 1
|
||||
self.tracked_objects[id]["motionless_count"] = 0
|
||||
self.stationary_box_history[id] = []
|
||||
self.stationary_classifier.on_active(id)
|
||||
|
||||
self.tracked_objects[id].update(obj)
|
||||
|
||||
@ -500,15 +467,6 @@ class NorfairTracker(ObjectTracker):
|
||||
) -> None:
|
||||
# Group detections by object type
|
||||
detections_by_type: dict[str, list[Detection]] = {}
|
||||
yuv_frame: np.ndarray | None = None
|
||||
|
||||
if self.ptz_metrics.autotracker_enabled.value or (
|
||||
self.detect_config.stationary.classifier
|
||||
and any(obj[0] == "car" for obj in detections)
|
||||
):
|
||||
yuv_frame = self.frame_manager.get(
|
||||
frame_name, self.camera_config.frame_shape_yuv
|
||||
)
|
||||
for obj in detections:
|
||||
label = obj[0]
|
||||
if label not in detections_by_type:
|
||||
@ -523,6 +481,9 @@ class NorfairTracker(ObjectTracker):
|
||||
|
||||
embedding = None
|
||||
if self.ptz_metrics.autotracker_enabled.value:
|
||||
yuv_frame = self.frame_manager.get(
|
||||
frame_name, self.camera_config.frame_shape_yuv
|
||||
)
|
||||
embedding = get_histogram(
|
||||
yuv_frame, obj[2][0], obj[2][1], obj[2][2], obj[2][3]
|
||||
)
|
||||
@ -614,11 +575,7 @@ class NorfairTracker(ObjectTracker):
|
||||
self.tracked_objects[id]["estimate"] = new_obj["estimate"]
|
||||
# else update it
|
||||
else:
|
||||
self.update(
|
||||
str(t.global_id),
|
||||
new_obj,
|
||||
yuv_frame if new_obj["label"] == "car" else None,
|
||||
)
|
||||
self.update(str(t.global_id), new_obj)
|
||||
|
||||
# clear expired tracks
|
||||
expired_ids = [k for k in self.track_id_map.keys() if k not in active_ids]
|
||||
|
||||
@ -1,202 +0,0 @@
|
||||
"""Tools for determining if an object is stationary."""
|
||||
|
||||
import logging
|
||||
from typing import Any, cast
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from scipy.ndimage import gaussian_filter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
THRESHOLD_KNOWN_ACTIVE_IOU = 0.2
|
||||
THRESHOLD_STATIONARY_CHECK_IOU = 0.6
|
||||
THRESHOLD_ACTIVE_CHECK_IOU = 0.9
|
||||
MAX_STATIONARY_HISTORY = 10
|
||||
|
||||
|
||||
class StationaryMotionClassifier:
|
||||
"""Fallback classifier to prevent false flips from stationary to active.
|
||||
|
||||
Uses appearance consistency on a fixed spatial region (historical median box)
|
||||
to detect actual movement, ignoring bounding box detection variations.
|
||||
"""
|
||||
|
||||
CROP_SIZE = 96
|
||||
NCC_KEEP_THRESHOLD = 0.90 # High correlation = keep stationary
|
||||
NCC_ACTIVE_THRESHOLD = 0.85 # Low correlation = consider active
|
||||
SHIFT_KEEP_THRESHOLD = 0.02 # Small shift = keep stationary
|
||||
SHIFT_ACTIVE_THRESHOLD = 0.04 # Large shift = consider active
|
||||
DRIFT_ACTIVE_THRESHOLD = 0.12 # Cumulative drift over 5 frames
|
||||
CHANGED_FRAMES_TO_FLIP = 2
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.anchor_crops: dict[str, np.ndarray] = {}
|
||||
self.anchor_boxes: dict[str, tuple[int, int, int, int]] = {}
|
||||
self.changed_counts: dict[str, int] = {}
|
||||
self.shift_histories: dict[str, list[float]] = {}
|
||||
|
||||
# Pre-compute Hanning window for phase correlation
|
||||
hann = np.hanning(self.CROP_SIZE).astype(np.float64)
|
||||
self._hann2d = np.outer(hann, hann)
|
||||
|
||||
def reset(self, id: str) -> None:
|
||||
logger.debug("StationaryMotionClassifier.reset: id=%s", id)
|
||||
if id in self.anchor_crops:
|
||||
del self.anchor_crops[id]
|
||||
if id in self.anchor_boxes:
|
||||
del self.anchor_boxes[id]
|
||||
self.changed_counts[id] = 0
|
||||
self.shift_histories[id] = []
|
||||
|
||||
def _extract_y_crop(
|
||||
self, yuv_frame: np.ndarray, box: tuple[int, int, int, int]
|
||||
) -> np.ndarray:
|
||||
"""Extract and normalize Y-plane crop from bounding box."""
|
||||
y_height = yuv_frame.shape[0] // 3 * 2
|
||||
width = yuv_frame.shape[1]
|
||||
x1 = max(0, min(width - 1, box[0]))
|
||||
y1 = max(0, min(y_height - 1, box[1]))
|
||||
x2 = max(0, min(width - 1, box[2]))
|
||||
y2 = max(0, min(y_height - 1, box[3]))
|
||||
|
||||
if x2 <= x1:
|
||||
x2 = min(width - 1, x1 + 1)
|
||||
if y2 <= y1:
|
||||
y2 = min(y_height - 1, y1 + 1)
|
||||
|
||||
# Extract Y-plane crop, resize, and blur
|
||||
y_plane = yuv_frame[0:y_height, 0:width]
|
||||
crop = y_plane[y1:y2, x1:x2]
|
||||
crop_resized = cv2.resize(
|
||||
crop, (self.CROP_SIZE, self.CROP_SIZE), interpolation=cv2.INTER_AREA
|
||||
)
|
||||
result = cast(np.ndarray[Any, Any], gaussian_filter(crop_resized, sigma=0.5))
|
||||
logger.debug(
|
||||
"_extract_y_crop: box=%s clamped=(%d,%d,%d,%d) crop_shape=%s",
|
||||
box,
|
||||
x1,
|
||||
y1,
|
||||
x2,
|
||||
y2,
|
||||
crop.shape if "crop" in locals() else None,
|
||||
)
|
||||
return result
|
||||
|
||||
def ensure_anchor(
|
||||
self, id: str, yuv_frame: np.ndarray, median_box: tuple[int, int, int, int]
|
||||
) -> None:
|
||||
"""Initialize anchor crop from stable median box when object becomes stationary."""
|
||||
if id not in self.anchor_crops:
|
||||
self.anchor_boxes[id] = median_box
|
||||
self.anchor_crops[id] = self._extract_y_crop(yuv_frame, median_box)
|
||||
self.changed_counts[id] = 0
|
||||
self.shift_histories[id] = []
|
||||
logger.debug(
|
||||
"ensure_anchor: initialized id=%s median_box=%s crop_shape=%s",
|
||||
id,
|
||||
median_box,
|
||||
self.anchor_crops[id].shape,
|
||||
)
|
||||
|
||||
def on_active(self, id: str) -> None:
|
||||
"""Reset state when object becomes active to allow re-anchoring."""
|
||||
logger.debug("on_active: id=%s became active; resetting state", id)
|
||||
self.reset(id)
|
||||
|
||||
def evaluate(
|
||||
self, id: str, yuv_frame: np.ndarray, current_box: tuple[int, int, int, int]
|
||||
) -> bool:
|
||||
"""Return True to keep stationary, False to flip to active.
|
||||
|
||||
Compares the same spatial region (historical median box) across frames
|
||||
to detect actual movement, ignoring bounding box variations.
|
||||
"""
|
||||
|
||||
if id not in self.anchor_crops or id not in self.anchor_boxes:
|
||||
logger.debug("evaluate: id=%s has no anchor; default keep stationary", id)
|
||||
return True
|
||||
|
||||
# Compare same spatial region across frames
|
||||
anchor_box = self.anchor_boxes[id]
|
||||
anchor_crop = self.anchor_crops[id]
|
||||
curr_crop = self._extract_y_crop(yuv_frame, anchor_box)
|
||||
|
||||
# Compute appearance and motion metrics
|
||||
ncc = cv2.matchTemplate(curr_crop, anchor_crop, cv2.TM_CCOEFF_NORMED)[0, 0]
|
||||
a64 = anchor_crop.astype(np.float64) * self._hann2d
|
||||
c64 = curr_crop.astype(np.float64) * self._hann2d
|
||||
(shift_x, shift_y), _ = cv2.phaseCorrelate(a64, c64)
|
||||
shift_norm = float(np.hypot(shift_x, shift_y)) / float(self.CROP_SIZE)
|
||||
|
||||
logger.debug(
|
||||
"evaluate: id=%s metrics ncc=%.4f shift_norm=%.4f (shift_x=%.3f, shift_y=%.3f)",
|
||||
id,
|
||||
float(ncc),
|
||||
shift_norm,
|
||||
float(shift_x),
|
||||
float(shift_y),
|
||||
)
|
||||
|
||||
# Update rolling shift history
|
||||
history = self.shift_histories.get(id, [])
|
||||
history.append(shift_norm)
|
||||
if len(history) > 5:
|
||||
history = history[-5:]
|
||||
self.shift_histories[id] = history
|
||||
drift_sum = float(sum(history))
|
||||
|
||||
logger.debug(
|
||||
"evaluate: id=%s history_len=%d last_shift=%.4f drift_sum=%.4f",
|
||||
id,
|
||||
len(history),
|
||||
history[-1] if history else -1.0,
|
||||
drift_sum,
|
||||
)
|
||||
|
||||
# Early exit for clear stationary case
|
||||
if ncc >= self.NCC_KEEP_THRESHOLD and shift_norm < self.SHIFT_KEEP_THRESHOLD:
|
||||
self.changed_counts[id] = 0
|
||||
logger.debug(
|
||||
"evaluate: id=%s early-stationary keep=True (ncc>=%.2f and shift<%.2f)",
|
||||
id,
|
||||
self.NCC_KEEP_THRESHOLD,
|
||||
self.SHIFT_KEEP_THRESHOLD,
|
||||
)
|
||||
return True
|
||||
|
||||
# Check for movement indicators
|
||||
movement_detected = (
|
||||
ncc < self.NCC_ACTIVE_THRESHOLD
|
||||
or shift_norm >= self.SHIFT_ACTIVE_THRESHOLD
|
||||
or drift_sum >= self.DRIFT_ACTIVE_THRESHOLD
|
||||
)
|
||||
|
||||
if movement_detected:
|
||||
cnt = self.changed_counts.get(id, 0) + 1
|
||||
self.changed_counts[id] = cnt
|
||||
if (
|
||||
cnt >= self.CHANGED_FRAMES_TO_FLIP
|
||||
or drift_sum >= self.DRIFT_ACTIVE_THRESHOLD
|
||||
):
|
||||
logger.debug(
|
||||
"evaluate: id=%s flip_to_active=True cnt=%d drift_sum=%.4f thresholds(changed>=%d drift>=%.2f)",
|
||||
id,
|
||||
cnt,
|
||||
drift_sum,
|
||||
self.CHANGED_FRAMES_TO_FLIP,
|
||||
self.DRIFT_ACTIVE_THRESHOLD,
|
||||
)
|
||||
return False
|
||||
logger.debug(
|
||||
"evaluate: id=%s movement_detected cnt=%d keep_until_cnt>=%d",
|
||||
id,
|
||||
cnt,
|
||||
self.CHANGED_FRAMES_TO_FLIP,
|
||||
)
|
||||
else:
|
||||
self.changed_counts[id] = 0
|
||||
logger.debug("evaluate: id=%s no_movement keep=True", id)
|
||||
|
||||
return True
|
||||
Loading…
x
Reference in New Issue
Block a user