Update ROCm to 6.4.1 (#18364)

* Update rocm to 6.4.1

* Quick fix

docker/rocm/Dockerfile

@@ -2,7 +2,7 @@
 
 # https://askubuntu.com/questions/972516/debian-frontend-environment-variable
 ARG DEBIAN_FRONTEND=noninteractive
-ARG ROCM=6.4.0
+ARG ROCM=1
 ARG AMDGPU=gfx900
 ARG HSA_OVERRIDE_GFX_VERSION
 ARG HSA_OVERRIDE
@@ -13,12 +13,12 @@ FROM wget AS rocm
 ARG ROCM
 ARG AMDGPU
 
-RUN apt update && \
+RUN apt update -qq && \
     apt install -y wget gpg && \
-    wget -O rocm.deb https://repo.radeon.com/amdgpu-install/6.4/ubuntu/jammy/amdgpu-install_6.4.60400-1_all.deb && \
+    wget -O rocm.deb https://repo.radeon.com/amdgpu-install/6.4.1/ubuntu/jammy/amdgpu-install_6.4.60401-1_all.deb && \
     apt install -y ./rocm.deb && \
     apt update && \
-    apt install -y rocm
+    apt install -qq -y rocm
 
 RUN mkdir -p /opt/rocm-dist/opt/rocm-$ROCM/lib
 RUN cd /opt/rocm-$ROCM/lib && \

docker/rocm/requirements-wheels-rocm.txt

@@ -1 +1 @@
-onnxruntime-rocm @ https://github.com/NickM-27/frigate-onnxruntime-rocm/releases/download/v6.4.0/onnxruntime_rocm-1.21.1-cp311-cp311-linux_x86_64.whl
+onnxruntime-rocm @ https://github.com/NickM-27/frigate-onnxruntime-rocm/releases/download/v6.4.1/onnxruntime_rocm-1.21.1-cp311-cp311-linux_x86_64.whl

docker/rocm/rocm.hcl

@@ -2,7 +2,7 @@ variable "AMDGPU" {
   default = "gfx900"
 }
 variable "ROCM" {
-  default = "6.4.0"
+  default = "6.4.1"
 }
 variable "HSA_OVERRIDE_GFX_VERSION" {
   default = ""

frigate/api/defs/query/media_query_parameters.py

@@ -1,7 +1,8 @@
 from enum import Enum
-from typing import Optional
+from typing import Optional, Union
 
 from pydantic import BaseModel
+from pydantic.json_schema import SkipJsonSchema
 
 
 class Extension(str, Enum):
@@ -46,3 +47,10 @@ class MediaMjpegFeedQueryParams(BaseModel):
 class MediaRecordingsSummaryQueryParams(BaseModel):
     timezone: str = "utc"
     cameras: Optional[str] = "all"
+
+
+class MediaRecordingsAvailabilityQueryParams(BaseModel):
+    cameras: str = "all"
+    before: Union[float, SkipJsonSchema[None]] = None
+    after: Union[float, SkipJsonSchema[None]] = None
+    scale: int = 30

frigate/api/media.py

@@ -8,6 +8,7 @@ import os
 import subprocess as sp
 import time
 from datetime import datetime, timedelta, timezone
+from functools import reduce
 from pathlib import Path as FilePath
 from typing import Any
 from urllib.parse import unquote
@@ -19,7 +20,7 @@ from fastapi import APIRouter, Path, Query, Request, Response
 from fastapi.params import Depends
 from fastapi.responses import FileResponse, JSONResponse, StreamingResponse
 from pathvalidate import sanitize_filename
-from peewee import DoesNotExist, fn
+from peewee import DoesNotExist, fn, operator
 from tzlocal import get_localzone_name
 
 from frigate.api.defs.query.media_query_parameters import (
@@ -27,6 +28,7 @@ from frigate.api.defs.query.media_query_parameters import (
     MediaEventsSnapshotQueryParams,
     MediaLatestFrameQueryParams,
     MediaMjpegFeedQueryParams,
+    MediaRecordingsAvailabilityQueryParams,
     MediaRecordingsSummaryQueryParams,
 )
 from frigate.api.defs.tags import Tags
@@ -542,6 +544,66 @@ def recordings(
     return JSONResponse(content=list(recordings))
 
 
+@router.get("/recordings/unavailable", response_model=list[dict])
+def no_recordings(params: MediaRecordingsAvailabilityQueryParams = Depends()):
+    """Get time ranges with no recordings."""
+    cameras = params.cameras
+    before = params.before or datetime.datetime.now().timestamp()
+    after = (
+        params.after
+        or (datetime.datetime.now() - datetime.timedelta(hours=1)).timestamp()
+    )
+    scale = params.scale
+
+    clauses = [(Recordings.start_time > after) & (Recordings.end_time < before)]
+    if cameras != "all":
+        camera_list = cameras.split(",")
+        clauses.append((Recordings.camera << camera_list))
+
+    # Get recording start times
+    data: list[Recordings] = (
+        Recordings.select(Recordings.start_time, Recordings.end_time)
+        .where(reduce(operator.and_, clauses))
+        .order_by(Recordings.start_time.asc())
+        .dicts()
+        .iterator()
+    )
+
+    # Convert recordings to list of (start, end) tuples
+    recordings = [(r["start_time"], r["end_time"]) for r in data]
+
+    # Generate all time segments
+    current = after
+    no_recording_segments = []
+    current_start = None
+
+    while current < before:
+        segment_end = current + scale
+        # Check if segment overlaps with any recording
+        has_recording = any(
+            start <= segment_end and end >= current for start, end in recordings
+        )
+        if not has_recording:
+            if current_start is None:
+                current_start = current  # Start a new gap
+        else:
+            if current_start is not None:
+                # End the current gap and append it
+                no_recording_segments.append(
+                    {"start_time": int(current_start), "end_time": int(current)}
+                )
+                current_start = None
+        current = segment_end
+
+    # Append the last gap if it exists
+    if current_start is not None:
+        no_recording_segments.append(
+            {"start_time": int(current_start), "end_time": int(before)}
+        )
+
+    return JSONResponse(content=no_recording_segments)
+
+
 @router.get(
     "/{camera_name}/start/{start_ts}/end/{end_ts}/clip.mp4",
     description="For iOS devices, use the master.m3u8 HLS link instead of clip.mp4. Safari does not reliably process progressive mp4 files.",

frigate/comms/dispatcher.py

@@ -289,7 +289,10 @@ class Dispatcher:
                 logger.info(f"Turning off detection for {camera_name}")
                 detect_settings.enabled = False
 
-        self.config_updater.publish(f"config/detect/{camera_name}", detect_settings)
+        self.config_updater.publish_update(
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.detect, camera_name),
+            detect_settings,
+        )
         self.publish(f"{camera_name}/detect/state", payload, retain=True)
 
     def _on_enabled_command(self, camera_name: str, payload: str) -> None:

frigate/config/classification.py

@@ -34,10 +34,37 @@ class BirdClassificationConfig(FrigateBaseModel):
     )
 
 
+class CustomClassificationStateCameraConfig(FrigateBaseModel):
+    crop: list[int, int, int, int] = Field(
+        title="Crop of image frame on this camera to run classification on."
+    )
+
+
+class CustomClassificationStateConfig(FrigateBaseModel):
+    cameras: Dict[str, CustomClassificationStateCameraConfig] = Field(
+        title="Cameras to run classification on."
+    )
+
+
+class CustomClassificationObjectConfig(FrigateBaseModel):
+    objects: list[str] = Field(title="Object types to classify.")
+
+
+class CustomClassificationConfig(FrigateBaseModel):
+    enabled: bool = Field(default=True, title="Enable running the model.")
+    model_path: str = Field(title="Path to custom classification tflite model.")
+    labelmap_path: str = Field(title="Path to custom classification model labelmap.")
+    object_config: CustomClassificationObjectConfig | None = Field(default=None)
+    state_config: CustomClassificationStateConfig | None = Field(default=None)
+
+
 class ClassificationConfig(FrigateBaseModel):
     bird: BirdClassificationConfig = Field(
         default_factory=BirdClassificationConfig, title="Bird classification config."
     )
+    custom: Dict[str, CustomClassificationConfig] = Field(
+        default={}, title="Custom Classification Model Configs."
+    )
 
 
 class SemanticSearchConfig(FrigateBaseModel):

frigate/data_processing/real_time/custom_classification.py

@@ -0,0 +1,178 @@
+"""Real time processor that works with classification tflite models."""
+
+import logging
+from typing import Any
+
+import cv2
+import numpy as np
+
+from frigate.comms.event_metadata_updater import (
+    EventMetadataPublisher,
+    EventMetadataTypeEnum,
+)
+from frigate.config import FrigateConfig
+from frigate.config.classification import CustomClassificationConfig
+from frigate.util.builtin import load_labels
+from frigate.util.object import calculate_region
+
+from ..types import DataProcessorMetrics
+from .api import RealTimeProcessorApi
+
+try:
+    from tflite_runtime.interpreter import Interpreter
+except ModuleNotFoundError:
+    from tensorflow.lite.python.interpreter import Interpreter
+
+logger = logging.getLogger(__name__)
+
+
+class CustomStateClassificationProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.model_config = model_config
+        self.interpreter: Interpreter = None
+        self.tensor_input_details: dict[str, Any] = None
+        self.tensor_output_details: dict[str, Any] = None
+        self.labelmap: dict[int, str] = {}
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        self.interpreter = Interpreter(
+            model_path=self.model_config.model_path,
+            num_threads=2,
+        )
+        self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(self.model_config.labelmap_path, prefill=0)
+
+    def process_frame(self, frame_data: dict[str, Any], frame: np.ndarray):
+        camera = frame_data.get("camera")
+        if camera not in self.model_config.state_config.cameras:
+            return
+
+        camera_config = self.model_config.state_config.cameras[camera]
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            camera_config.crop[0],
+            camera_config.crop[1],
+            camera_config.crop[2],
+            camera_config.crop[3],
+            224,
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        input = rgb[
+            y:y2,
+            x:x2,
+        ]
+
+        if input.shape != (224, 224):
+            input = cv2.resize(input, (224, 224))
+
+        input = np.expand_dims(input, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        print(f"the gate res is {res}")
+        probs = res / res.sum(axis=0)
+        best_id = np.argmax(probs)
+        score = round(probs[best_id], 2)
+
+        print(f"got {self.labelmap[best_id]} with score {score}")
+
+    def handle_request(self, topic, request_data):
+        return None
+
+    def expire_object(self, object_id, camera):
+        pass
+
+
+class CustomObjectClassificationProcessor(RealTimeProcessorApi):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        model_config: CustomClassificationConfig,
+        sub_label_publisher: EventMetadataPublisher,
+        metrics: DataProcessorMetrics,
+    ):
+        super().__init__(config, metrics)
+        self.model_config = model_config
+        self.interpreter: Interpreter = None
+        self.sub_label_publisher = sub_label_publisher
+        self.tensor_input_details: dict[str, Any] = None
+        self.tensor_output_details: dict[str, Any] = None
+        self.detected_objects: dict[str, float] = {}
+        self.labelmap: dict[int, str] = {}
+        self.__build_detector()
+
+    def __build_detector(self) -> None:
+        self.interpreter = Interpreter(
+            model_path=self.model_config.model_path,
+            num_threads=2,
+        )
+        self.interpreter.allocate_tensors()
+        self.tensor_input_details = self.interpreter.get_input_details()
+        self.tensor_output_details = self.interpreter.get_output_details()
+        self.labelmap = load_labels(self.model_config.labelmap_path, prefill=0)
+
+    def process_frame(self, obj_data, frame):
+        if obj_data["label"] not in self.model_config.object_config.objects:
+            return
+
+        x, y, x2, y2 = calculate_region(
+            frame.shape,
+            obj_data["box"][0],
+            obj_data["box"][1],
+            obj_data["box"][2],
+            obj_data["box"][3],
+            224,
+            1.0,
+        )
+
+        rgb = cv2.cvtColor(frame, cv2.COLOR_YUV2RGB_I420)
+        input = rgb[
+            y:y2,
+            x:x2,
+        ]
+
+        if input.shape != (224, 224):
+            input = cv2.resize(input, (224, 224))
+
+        input = np.expand_dims(input, axis=0)
+        self.interpreter.set_tensor(self.tensor_input_details[0]["index"], input)
+        self.interpreter.invoke()
+        res: np.ndarray = self.interpreter.get_tensor(
+            self.tensor_output_details[0]["index"]
+        )[0]
+        probs = res / res.sum(axis=0)
+        best_id = np.argmax(probs)
+
+        score = round(probs[best_id], 2)
+
+        previous_score = self.detected_objects.get(obj_data["id"], 0.0)
+
+        if score <= previous_score:
+            logger.debug(f"Score {score} is worse than previous score {previous_score}")
+            return
+
+        self.sub_label_publisher.publish(
+            EventMetadataTypeEnum.sub_label,
+            (obj_data["id"], self.labelmap[best_id], score),
+        )
+        self.detected_objects[obj_data["id"]] = score
+
+    def handle_request(self, topic, request_data):
+        return None
+
+    def expire_object(self, object_id, camera):
+        if object_id in self.detected_objects:
+            self.detected_objects.pop(object_id)

frigate/embeddings/maintainer.py

@@ -42,6 +42,10 @@ from frigate.data_processing.post.license_plate import (
 )
 from frigate.data_processing.real_time.api import RealTimeProcessorApi
 from frigate.data_processing.real_time.bird import BirdRealTimeProcessor
+from frigate.data_processing.real_time.custom_classification import (
+    CustomObjectClassificationProcessor,
+    CustomStateClassificationProcessor,
+)
 from frigate.data_processing.real_time.face import FaceRealTimeProcessor
 from frigate.data_processing.real_time.license_plate import (
     LicensePlateRealTimeProcessor,
@@ -143,6 +147,18 @@ class EmbeddingMaintainer(threading.Thread):
                 )
             )
 
+        for model in self.config.classification.custom.values():
+            self.realtime_processors.append(
+                CustomStateClassificationProcessor(self.config, model, self.metrics)
+                if model.state_config != None
+                else CustomObjectClassificationProcessor(
+                    self.config,
+                    model,
+                    self.event_metadata_publisher,
+                    self.metrics,
+                )
+            )
+
         # post processors
         self.post_processors: list[PostProcessorApi] = []
 
@@ -172,7 +188,7 @@ class EmbeddingMaintainer(threading.Thread):
             self._process_requests()
             self._process_updates()
             self._process_recordings_updates()
-            self._process_dedicated_lpr()
+            self._process_frame_updates()
             self._expire_dedicated_lpr()
             self._process_finalized()
             self._process_event_metadata()
@@ -449,7 +465,7 @@ class EmbeddingMaintainer(threading.Thread):
                 event_id, RegenerateDescriptionEnum(source)
             )
 
-    def _process_dedicated_lpr(self) -> None:
+    def _process_frame_updates(self) -> None:
         """Process event updates"""
         (topic, data) = self.detection_subscriber.check_for_update()
 
@@ -458,7 +474,7 @@ class EmbeddingMaintainer(threading.Thread):
 
         camera, frame_name, _, _, motion_boxes, _ = data
 
-        if not camera or not self.config.lpr.enabled or len(motion_boxes) == 0:
+        if not camera or len(motion_boxes) == 0:
             return
 
         camera_config = self.config.cameras[camera]
@@ -466,8 +482,8 @@ class EmbeddingMaintainer(threading.Thread):
         if (
             camera_config.type != CameraTypeEnum.lpr
             or "license_plate" in camera_config.objects.track
-        ):
+        ) and len(self.config.classification.custom) == 0:
-            # we're not a dedicated lpr camera or we are one but we're using frigate+
+            # no active features that use this data
             return
 
         try:
@@ -487,6 +503,9 @@ class EmbeddingMaintainer(threading.Thread):
             if isinstance(processor, LicensePlateRealTimeProcessor):
                 processor.process_frame(camera, yuv_frame, True)
 
+            if isinstance(processor, CustomStateClassificationProcessor):
+                processor.process_frame({"camera": camera}, yuv_frame)
+
         self.frame_manager.close(frame_name)
 
     def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:

web/src/components/timeline/MotionReviewTimeline.tsx

@@ -17,6 +17,7 @@ import {
   VirtualizedMotionSegments,
   VirtualizedMotionSegmentsRef,
 } from "./VirtualizedMotionSegments";
+import { RecordingSegment } from "@/types/record";
 
 export type MotionReviewTimelineProps = {
   segmentDuration: number;
@@ -38,6 +39,7 @@ export type MotionReviewTimelineProps = {
   setExportEndTime?: React.Dispatch<React.SetStateAction<number>>;
   events: ReviewSegment[];
   motion_events: MotionData[];
+  noRecordingRanges?: RecordingSegment[];
   contentRef: RefObject<HTMLDivElement>;
   timelineRef?: RefObject<HTMLDivElement>;
   onHandlebarDraggingChange?: (isDragging: boolean) => void;
@@ -66,6 +68,7 @@ export function MotionReviewTimeline({
   setExportEndTime,
   events,
   motion_events,
+  noRecordingRanges,
   contentRef,
   timelineRef,
   onHandlebarDraggingChange,
@@ -97,6 +100,17 @@ export function MotionReviewTimeline({
     motion_events,
   );
 
+  const getRecordingAvailability = useCallback(
+    (time: number): boolean | undefined => {
+      if (!noRecordingRanges?.length) return undefined;
+
+      return !noRecordingRanges.some(
+        (range) => time >= range.start_time && time < range.end_time,
+      );
+    },
+    [noRecordingRanges],
+  );
+
   const segmentTimes = useMemo(() => {
     const segments = [];
     let segmentTime = timelineStartAligned;
@@ -206,6 +220,7 @@ export function MotionReviewTimeline({
         dense={dense}
         motionOnly={motionOnly}
         getMotionSegmentValue={getMotionSegmentValue}
+        getRecordingAvailability={getRecordingAvailability}
       />
     </ReviewTimeline>
   );

web/src/components/timeline/MotionSegment.tsx

@@ -15,6 +15,7 @@ type MotionSegmentProps = {
   timestampSpread: number;
   firstHalfMotionValue: number;
   secondHalfMotionValue: number;
+  hasRecording?: boolean;
   motionOnly: boolean;
   showMinimap: boolean;
   minimapStartTime?: number;
@@ -31,6 +32,7 @@ export function MotionSegment({
   timestampSpread,
   firstHalfMotionValue,
   secondHalfMotionValue,
+  hasRecording,
   motionOnly,
   showMinimap,
   minimapStartTime,
@@ -176,6 +178,12 @@ export function MotionSegment({
             segmentClasses,
             severity[0] && "bg-gradient-to-r",
             severity[0] && severityColorsBg[severity[0]],
+            // TODO: will update this for 0.17
+            false &&
+              hasRecording == false &&
+              firstHalfMotionValue == 0 &&
+              secondHalfMotionValue == 0 &&
+              "bg-slashes",
           )}
           onClick={segmentClick}
           onTouchEnd={(event) => handleTouchStart(event, segmentClick)}

web/src/components/timeline/VirtualizedMotionSegments.tsx

@@ -24,6 +24,7 @@ type VirtualizedMotionSegmentsProps = {
   dense: boolean;
   motionOnly: boolean;
   getMotionSegmentValue: (timestamp: number) => number;
+  getRecordingAvailability: (timestamp: number) => boolean | undefined;
 };
 
 export interface VirtualizedMotionSegmentsRef {
@@ -55,6 +56,7 @@ export const VirtualizedMotionSegments = forwardRef<
       dense,
       motionOnly,
       getMotionSegmentValue,
+      getRecordingAvailability,
     },
     ref,
   ) => {
@@ -154,6 +156,8 @@ export const VirtualizedMotionSegments = forwardRef<
               (item.end_time ?? segmentTime) >= motionEnd),
         );
 
+        const hasRecording = getRecordingAvailability(segmentTime);
+
         if ((!segmentMotion || overlappingReviewItems) && motionOnly) {
           return null; // Skip rendering this segment in motion only mode
         }
@@ -172,6 +176,7 @@ export const VirtualizedMotionSegments = forwardRef<
               events={events}
               firstHalfMotionValue={firstHalfMotionValue}
               secondHalfMotionValue={secondHalfMotionValue}
+              hasRecording={hasRecording}
               segmentDuration={segmentDuration}
               segmentTime={segmentTime}
               timestampSpread={timestampSpread}
@@ -189,6 +194,7 @@ export const VirtualizedMotionSegments = forwardRef<
       [
         events,
         getMotionSegmentValue,
+        getRecordingAvailability,
         motionOnly,
         segmentDuration,
         showMinimap,

web/src/views/recording/RecordingView.tsx

@@ -43,7 +43,11 @@ import Logo from "@/components/Logo";
 import { Skeleton } from "@/components/ui/skeleton";
 import { FaVideo } from "react-icons/fa";
 import { VideoResolutionType } from "@/types/live";
-import { ASPECT_VERTICAL_LAYOUT, ASPECT_WIDE_LAYOUT } from "@/types/record";
+import {
+  ASPECT_VERTICAL_LAYOUT,
+  ASPECT_WIDE_LAYOUT,
+  RecordingSegment,
+} from "@/types/record";
 import { useResizeObserver } from "@/hooks/resize-observer";
 import { cn } from "@/lib/utils";
 import { useFullscreen } from "@/hooks/use-fullscreen";
@@ -808,6 +812,16 @@ function Timeline({
     },
   ]);
 
+  const { data: noRecordings } = useSWR<RecordingSegment[]>([
+    "recordings/unavailable",
+    {
+      before: timeRange.before,
+      after: timeRange.after,
+      scale: Math.round(zoomSettings.segmentDuration / 2),
+      cameras: mainCamera,
+    },
+  ]);
+
   const [exportStart, setExportStartTime] = useState<number>(0);
   const [exportEnd, setExportEndTime] = useState<number>(0);
 
@@ -853,6 +867,7 @@ function Timeline({
             setHandlebarTime={setCurrentTime}
             events={mainCameraReviewItems}
             motion_events={motionData ?? []}
+            noRecordingRanges={noRecordings ?? []}
             contentRef={contentRef}
             onHandlebarDraggingChange={(scrubbing) => setScrubbing(scrubbing)}
             isZooming={isZooming}

web/tailwind.config.cjs

@@ -42,6 +42,10 @@ module.exports = {
         wide: "32 / 9",
         tall: "8 / 9",
       },
+      backgroundImage: {
+        slashes:
+          "repeating-linear-gradient(45deg, hsl(var(--primary-variant) / 0.2), hsl(var(--primary-variant) / 0.2) 2px, transparent 2px, transparent 8px)",
+      },
       colors: {
         border: "hsl(var(--border))",
         input: "hsl(var(--input))",