Use preview frames for Review Descriptions (#19450)

* Use preview frames for genai

* Cleanup

* Adjust

docs/docs/configuration/genai.md

@@ -39,7 +39,7 @@ By default, descriptions will be generated for all tracked objects and all zones
 
 Optionally, you can generate the description using a snapshot (if enabled) by setting `use_snapshot` to `True`. By default, this is set to `False`, which sends the uncompressed images from the `detect` stream collected over the object's lifetime to the model. Once the object lifecycle ends, only a single compressed and cropped thumbnail is saved with the tracked object. Using a snapshot might be useful when you want to _regenerate_ a tracked object's description as it will provide the AI with a higher-quality image (typically downscaled by the AI itself) than the cropped/compressed thumbnail. Using a snapshot otherwise has a trade-off in that only a single image is sent to your provider, which will limit the model's ability to determine object movement or direction.
 
-Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/genai/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_namegenaiset).
+Generative AI can also be toggled dynamically for a camera via MQTT with the topic `frigate/<camera_name>/object_descriptions/set`. See the [MQTT documentation](/integrations/mqtt/#frigatecamera_nameobjectdescriptionsset).
 
 ## Ollama
 

docs/docs/integrations/mqtt.md

@@ -411,13 +411,21 @@ Topic to turn review detections for a camera on or off. Expected values are `ON`
 
 Topic with current state of review detections for a camera. Published values are `ON` and `OFF`.
 
-### `frigate/<camera_name>/genai/set`
+### `frigate/<camera_name>/object_descriptions/set`
 
-Topic to turn generative AI for a camera on or off. Expected values are `ON` and `OFF`.
+Topic to turn generative AI object descriptions for a camera on or off. Expected values are `ON` and `OFF`.
 
-### `frigate/<camera_name>/genai/state`
+### `frigate/<camera_name>/object_descriptions/state`
 
-Topic with current state of generative AI for a camera. Published values are `ON` and `OFF`.
+Topic with current state of generative AI object descriptions for a camera. Published values are `ON` and `OFF`.
+
+### `frigate/<camera_name>/review_descriptions/set`
+
+Topic to turn generative AI review descriptions for a camera on or off. Expected values are `ON` and `OFF`.
+
+### `frigate/<camera_name>/review_descriptions/state`
+
+Topic with current state of generative AI review descriptions for a camera. Published values are `ON` and `OFF`.
 
 ### `frigate/<camera_name>/birdseye/set`
 

frigate/comms/dispatcher.py

@@ -26,6 +26,7 @@ from frigate.const import (
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
     UPDATE_EVENT_DESCRIPTION,
     UPDATE_MODEL_STATE,
+    UPDATE_REVIEW_DESCRIPTION,
     UPSERT_REVIEW_SEGMENT,
 )
 from frigate.models import Event, Previews, Recordings, ReviewSegment
@@ -74,7 +75,8 @@ class Dispatcher:
             "birdseye_mode": self._on_birdseye_mode_command,
             "review_alerts": self._on_alerts_command,
             "review_detections": self._on_detections_command,
-            "genai": self._on_genai_command,
+            "object_descriptions": self._on_object_description_command,
+            "review_descriptions": self._on_review_description_command,
         }
         self._global_settings_handlers: dict[str, Callable] = {
             "notifications": self._on_global_notification_command,
@@ -149,6 +151,14 @@ class Dispatcher:
                 ),
             )
 
+        def handle_update_review_description() -> None:
+            final_data = payload["after"]
+            ReviewSegment.insert(final_data).on_conflict(
+                conflict_target=[ReviewSegment.id],
+                update=final_data,
+            ).execute()
+            self.publish("reviews", json.dumps(payload))
+
         def handle_update_model_state() -> None:
             if payload:
                 model = payload["model"]
@@ -209,7 +219,12 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
-                    "genai": self.config.cameras[camera].objects.genai.enabled,
+                    "object_descriptions": self.config.cameras[
+                        camera
+                    ].objects.genai.enabled,
+                    "review_descriptions": self.config.cameras[
+                        camera
+                    ].review.genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -232,6 +247,7 @@ class Dispatcher:
             CLEAR_ONGOING_REVIEW_SEGMENTS: handle_clear_ongoing_review_segments,
             UPDATE_CAMERA_ACTIVITY: handle_update_camera_activity,
             UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
+            UPDATE_REVIEW_DESCRIPTION: handle_update_review_description,
             UPDATE_MODEL_STATE: handle_update_model_state,
             UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
             UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,
@@ -742,8 +758,8 @@ class Dispatcher:
         )
         self.publish(f"{camera_name}/review_detections/state", payload, retain=True)
 
-    def _on_genai_command(self, camera_name: str, payload: str) -> None:
-        """Callback for GenAI topic."""
+    def _on_object_description_command(self, camera_name: str, payload: str) -> None:
+        """Callback for object description topic."""
         genai_settings = self.config.cameras[camera_name].objects.genai
 
         if payload == "ON":
@@ -754,15 +770,40 @@ class Dispatcher:
                 return
 
             if not genai_settings.enabled:
-                logger.info(f"Turning on GenAI for {camera_name}")
+                logger.info(f"Turning on object descriptions for {camera_name}")
                 genai_settings.enabled = True
         elif payload == "OFF":
             if genai_settings.enabled:
-                logger.info(f"Turning off GenAI for {camera_name}")
+                logger.info(f"Turning off object descriptions for {camera_name}")
                 genai_settings.enabled = False
 
         self.config_updater.publish_update(
-            CameraConfigUpdateTopic(CameraConfigUpdateEnum.genai, camera_name),
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.object_genai, camera_name),
             genai_settings,
         )
-        self.publish(f"{camera_name}/genai/state", payload, retain=True)
+        self.publish(f"{camera_name}/object_descriptions/state", payload, retain=True)
+
+    def _on_review_description_command(self, camera_name: str, payload: str) -> None:
+        """Callback for review description topic."""
+        genai_settings = self.config.cameras[camera_name].review.genai
+
+        if payload == "ON":
+            if not self.config.cameras[camera_name].review.genai.enabled_in_config:
+                logger.error(
+                    "GenAI Alerts or Detections must be enabled in the config to be turned on via MQTT."
+                )
+                return
+
+            if not genai_settings.enabled:
+                logger.info(f"Turning on review descriptions for {camera_name}")
+                genai_settings.enabled = True
+        elif payload == "OFF":
+            if genai_settings.enabled:
+                logger.info(f"Turning off review descriptions for {camera_name}")
+                genai_settings.enabled = False
+
+        self.config_updater.publish_update(
+            CameraConfigUpdateTopic(CameraConfigUpdateEnum.review_genai, camera_name),
+            genai_settings,
+        )
+        self.publish(f"{camera_name}/review_descriptions/state", payload, retain=True)

frigate/comms/mqtt.py

@@ -123,10 +123,15 @@ class MqttClient(Communicator):
                 retain=True,
             )
             self.publish(
-                f"{camera_name}/genai/state",
+                f"{camera_name}/object_descriptions/state",
                 "ON" if camera.objects.genai.enabled_in_config else "OFF",
                 retain=True,
             )
+            self.publish(
+                f"{camera_name}/review_descriptions/state",
+                "ON" if camera.review.genai.enabled_in_config else "OFF",
+                retain=True,
+            )
 
         if self.config.notifications.enabled_in_config:
             self.publish(

frigate/comms/webpush.py

@@ -369,12 +369,22 @@ class WebPushClient(Communicator):
         sorted_objects.update(payload["after"]["data"]["sub_labels"])
 
         title = f"{titlecase(', '.join(sorted_objects).replace('_', ' '))}{' was' if state == 'end' else ''} detected in {titlecase(', '.join(payload['after']['data']['zones']).replace('_', ' '))}"
-        message = f"Detected on {titlecase(camera.replace('_', ' '))}"
         image = f"{payload['after']['thumb_path'].replace('/media/frigate', '')}"
+        ended = state == "end" or state == "genai"
+
+        if state == "genai" and payload["after"]["data"]["metadata"]:
+            message = payload["after"]["data"]["metadata"]["scene"]
+        else:
+            message = f"Detected on {titlecase(camera.replace('_', ' '))}"
+
+        if ended:
+            logger.debug(
+                f"Sending a notification with state {state} and message {message}"
+            )
 
         # if event is ongoing open to live view otherwise open to recordings view
-        direct_url = f"/review?id={reviewId}" if state == "end" else f"/#{camera}"
-        ttl = 3600 if state == "end" else 0
+        direct_url = f"/review?id={reviewId}" if ended else f"/#{camera}"
+        ttl = 3600 if ended else 0
 
         logger.debug(f"Sending push notification for {camera}, review ID {reviewId}")
 

frigate/config/camera/review.py

@@ -62,6 +62,22 @@ class DetectionsConfig(FrigateBaseModel):
         return v
 
 
+class GenAIReviewConfig(FrigateBaseModel):
+    enabled: bool = Field(
+        default=False,
+        title="Enable GenAI descriptions for review items.",
+    )
+    alerts: bool = Field(default=True, title="Enable GenAI for alerts.")
+    detections: bool = Field(default=False, title="Enable GenAI for detections.")
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
+    enabled_in_config: Optional[bool] = Field(
+        default=None, title="Keep track of original state of generative AI."
+    )
+
+
 class ReviewConfig(FrigateBaseModel):
     """Configure reviews"""
 
@@ -71,3 +87,6 @@ class ReviewConfig(FrigateBaseModel):
     detections: DetectionsConfig = Field(
         default_factory=DetectionsConfig, title="Review detections config."
     )
+    genai: GenAIReviewConfig = Field(
+        default_factory=GenAIReviewConfig, title="Review description genai config."
+    )

frigate/config/camera/updater.py

@@ -17,13 +17,14 @@ class CameraConfigUpdateEnum(str, Enum):
     birdseye = "birdseye"
     detect = "detect"
     enabled = "enabled"
-    genai = "genai"
     motion = "motion"  # includes motion and motion masks
     notifications = "notifications"
     objects = "objects"
+    object_genai = "object_genai"
     record = "record"
     remove = "remove"  # for removing a camera
     review = "review"
+    review_genai = "review_genai"
     semantic_search = "semantic_search"  # for semantic search triggers
     snapshots = "snapshots"
     zones = "zones"
@@ -98,7 +99,7 @@ class CameraConfigUpdateSubscriber:
             config.detect = updated_config
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
-        elif update_type == CameraConfigUpdateEnum.genai:
+        elif update_type == CameraConfigUpdateEnum.object_genai:
             config.objects.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
@@ -110,6 +111,8 @@ class CameraConfigUpdateSubscriber:
             config.record = updated_config
         elif update_type == CameraConfigUpdateEnum.review:
             config.review = updated_config
+        elif update_type == CameraConfigUpdateEnum.review_genai:
+            config.review.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.semantic_search:
             config.semantic_search = updated_config
         elif update_type == CameraConfigUpdateEnum.snapshots:

frigate/config/config.py

@@ -610,6 +610,9 @@ class FrigateConfig(FrigateBaseModel):
             camera_config.objects.genai.enabled_in_config = (
                 camera_config.objects.genai.enabled
             )
+            camera_config.review.genai.enabled_in_config = (
+                camera_config.review.genai.enabled
+            )
 
             # Add default filters
             object_keys = camera_config.objects.track

frigate/const.py

@@ -111,6 +111,7 @@ UPSERT_REVIEW_SEGMENT = "upsert_review_segment"
 CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments"
 UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 UPDATE_EVENT_DESCRIPTION = "update_event_description"
+UPDATE_REVIEW_DESCRIPTION = "update_review_description"
 UPDATE_MODEL_STATE = "update_model_state"
 UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
 UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout"

frigate/data_processing/post/review_descriptions.py

@@ -1,25 +1,190 @@
 """Post processor for review items to get descriptions."""
 
+import copy
+import datetime
 import logging
-from typing import Any
+import os
+import shutil
+import threading
+from pathlib import Path
 
+import cv2
+
+from frigate.comms.inter_process import InterProcessRequestor
+from frigate.config import FrigateConfig
+from frigate.const import CACHE_DIR, CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION
 from frigate.data_processing.types import PostProcessDataEnum
+from frigate.genai import GenAIClient
+from frigate.util.builtin import EventsPerSecond, InferenceSpeed
 
 from ..post.api import PostProcessorApi
+from ..types import DataProcessorMetrics
 
 logger = logging.getLogger(__name__)
 
 
 class ReviewDescriptionProcessor(PostProcessorApi):
-    def __init__(self, config, metrics):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+        client: GenAIClient,
+    ):
         super().__init__(config, metrics, None)
-        self.tracked_review_items: dict[str, list[Any]] = {}
+        self.requestor = requestor
+        self.metrics = metrics
+        self.genai_client = client
+        self.review_desc_speed = InferenceSpeed(self.metrics.review_desc_speed)
+        self.review_descs_dps = EventsPerSecond()
+        self.review_descs_dps.start()
 
     def process_data(self, data, data_type):
+        self.metrics.review_desc_dps.value = self.review_descs_dps.eps()
+
         if data_type != PostProcessDataEnum.review:
             return
 
-        logger.info(f"processor is looking at {data}")
+        camera = data["after"]["camera"]
+
+        if not self.config.cameras[camera].review.genai.enabled:
+            return
+
+        id = data["after"]["id"]
+
+        if data["type"] == "new" or data["type"] == "update":
+            return
+        else:
+            final_data = data["after"]
+
+            if (
+                final_data["severity"] == "alert"
+                and not self.config.cameras[camera].review.genai.alerts
+            ):
+                return
+            elif (
+                final_data["severity"] == "detection"
+                and not self.config.cameras[camera].review.genai.detections
+            ):
+                return
+
+            frames = self.get_cache_frames(
+                camera, final_data["start_time"], final_data["end_time"]
+            )
+
+            if not frames:
+                frames = [final_data["thumb_path"]]
+
+            thumbs = []
+
+            for idx, thumb_path in enumerate(frames):
+                thumb_data = cv2.imread(thumb_path)
+                ret, jpg = cv2.imencode(
+                    ".jpg", thumb_data, [int(cv2.IMWRITE_JPEG_QUALITY), 100]
+                )
+
+                if ret:
+                    thumbs.append(jpg.tobytes())
+
+                if self.config.cameras[
+                    data["after"]["camera"]
+                ].review.genai.debug_save_thumbnails:
+                    id = data["after"]["id"]
+                    Path(os.path.join(CLIPS_DIR, f"genai-requests/{id}")).mkdir(
+                        parents=True, exist_ok=True
+                    )
+                    shutil.copy(
+                        thumb_path,
+                        os.path.join(
+                            CLIPS_DIR,
+                            f"genai-requests/{id}/{idx}.webp",
+                        ),
+                    )
+
+            # kickoff analysis
+            self.review_descs_dps.update()
+            threading.Thread(
+                target=run_analysis,
+                args=(
+                    self.requestor,
+                    self.genai_client,
+                    self.review_desc_speed,
+                    camera,
+                    final_data,
+                    thumbs,
+                ),
+            ).start()
 
     def handle_request(self, request_data):
         pass
+
+    def get_cache_frames(
+        self, camera: str, start_time: float, end_time: float
+    ) -> list[str]:
+        preview_dir = os.path.join(CACHE_DIR, "preview_frames")
+        file_start = f"preview_{camera}"
+        start_file = f"{file_start}-{start_time}.webp"
+        end_file = f"{file_start}-{end_time}.webp"
+        all_frames = []
+
+        for file in sorted(os.listdir(preview_dir)):
+            if not file.startswith(file_start):
+                continue
+
+            if file < start_file:
+                continue
+
+            if file > end_file:
+                break
+
+            all_frames.append(os.path.join(preview_dir, file))
+
+        frame_count = len(all_frames)
+        if frame_count <= 10:
+            return all_frames
+
+        selected_frames = []
+        step_size = (frame_count - 1) / 9
+
+        for i in range(10):
+            index = round(i * step_size)
+            selected_frames.append(all_frames[index])
+
+        return selected_frames
+
+
+@staticmethod
+def run_analysis(
+    requestor: InterProcessRequestor,
+    genai_client: GenAIClient,
+    review_inference_speed: InferenceSpeed,
+    camera: str,
+    final_data: dict[str, str],
+    thumbs: list[bytes],
+) -> None:
+    start = datetime.datetime.now().timestamp()
+    metadata = genai_client.generate_review_description(
+        {
+            "camera": camera,
+            "objects": final_data["data"]["objects"],
+            "recognized_objects": final_data["data"]["sub_labels"],
+            "zones": final_data["data"]["zones"],
+            "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
+        },
+        thumbs,
+    )
+    review_inference_speed.update(datetime.datetime.now().timestamp() - start)
+
+    if not metadata:
+        return None
+
+    prev_data = copy.deepcopy(final_data)
+    final_data["data"]["metadata"] = metadata.model_dump()
+    requestor.send_data(
+        UPDATE_REVIEW_DESCRIPTION,
+        {
+            "type": "genai",
+            "before": {k: v for k, v in prev_data.items()},
+            "after": {k: v for k, v in final_data.items()},
+        },
+    )

frigate/data_processing/post/types.py

@@ -0,0 +1,15 @@
+from pydantic import BaseModel, Field
+
+
+class ReviewMetadata(BaseModel):
+    scene: str = Field(
+        description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence."
+    )
+    confidence: float = Field(
+        description="A float between 0 and 1 representing your overall confidence in this analysis."
+    )
+    potential_threat_level: int = Field(
+        ge=0,
+        le=3,
+        description="An integer representing the potential threat level (1-3). 1: Minor anomaly. 2: Moderate concern. 3: High threat. Only include this field if a clear security concern is observable; otherwise, omit it.",
+    )

frigate/data_processing/types.py

@@ -20,6 +20,8 @@ class DataProcessorMetrics:
     alpr_pps: Synchronized
     yolov9_lpr_speed: Synchronized
     yolov9_lpr_pps: Synchronized
+    review_desc_speed: Synchronized
+    review_desc_dps: Synchronized
     classification_speeds: dict[str, Synchronized]
     classification_cps: dict[str, Synchronized]
 
@@ -34,6 +36,8 @@ class DataProcessorMetrics:
         self.alpr_pps = manager.Value("d", 0.0)
         self.yolov9_lpr_speed = manager.Value("d", 0.0)
         self.yolov9_lpr_pps = manager.Value("d", 0.0)
+        self.review_desc_speed = manager.Value("d", 0.0)
+        self.review_desc_dps = manager.Value("d", 0.0)
         self.classification_speeds = manager.dict()
         self.classification_cps = manager.dict()
 

frigate/embeddings/maintainer.py

@@ -102,7 +102,8 @@ class EmbeddingMaintainer(threading.Thread):
             [
                 CameraConfigUpdateEnum.add,
                 CameraConfigUpdateEnum.remove,
-                CameraConfigUpdateEnum.genai,
+                CameraConfigUpdateEnum.object_genai,
+                CameraConfigUpdateEnum.review_genai,
                 CameraConfigUpdateEnum.semantic_search,
             ],
         )
@@ -151,6 +152,7 @@ class EmbeddingMaintainer(threading.Thread):
         self.frame_manager = SharedMemoryFrameManager()
 
         self.detected_license_plates: dict[str, dict[str, Any]] = {}
+        self.genai_client = get_genai_client(config)
 
         # model runners to share between realtime and post processors
         if self.config.lpr.enabled:
@@ -206,6 +208,13 @@ class EmbeddingMaintainer(threading.Thread):
         # post processors
         self.post_processors: list[PostProcessorApi] = []
 
+        if any(c.review.genai.enabled_in_config for c in self.config.cameras.values()):
+            self.post_processors.append(
+                ReviewDescriptionProcessor(
+                    self.config, self.requestor, self.metrics, self.genai_client
+                )
+            )
+
         if self.config.lpr.enabled:
             self.post_processors.append(
                 LicensePlatePostProcessor(
@@ -240,7 +249,6 @@ class EmbeddingMaintainer(threading.Thread):
         self.stop_event = stop_event
         self.tracked_events: dict[str, list[Any]] = {}
         self.early_request_sent: dict[str, bool] = {}
-        self.genai_client = get_genai_client(config)
 
         # recordings data
         self.recordings_available_through: dict[str, float] = {}
@@ -688,7 +696,7 @@ class EmbeddingMaintainer(threading.Thread):
         """Embed the description for an event."""
         camera_config = self.config.cameras[event.camera]
 
-        description = self.genai_client.generate_description(
+        description = self.genai_client.generate_object_description(
             camera_config, thumbnails, event
         )
 

frigate/genai/__init__.py

@@ -3,11 +3,13 @@
 import importlib
 import logging
 import os
-from typing import Optional
+import re
+from typing import Any, Optional
 
 from playhouse.shortcuts import model_to_dict
 
 from frigate.config import CameraConfig, FrigateConfig, GenAIConfig, GenAIProviderEnum
+from frigate.data_processing.post.types import ReviewMetadata
 from frigate.models import Event
 
 logger = logging.getLogger(__name__)
@@ -33,7 +35,63 @@ class GenAIClient:
         self.timeout = timeout
         self.provider = self._init_provider()
 
-    def generate_description(
+    def generate_review_description(
+        self, review_data: dict[str, Any], thumbnails: list[bytes]
+    ) -> ReviewMetadata | None:
+        """Generate a description for the review item activity."""
+        context_prompt = f"""
+        Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
+
+        Your task is to provide a **neutral, factual, and objective description** of the scene, while also:
+        - Clearly stating **what is happening** based on observable actions and movements.
+        - Including **reasonable, evidence-based inferences** about the likely activity or context, but only if directly supported by visible details.
+
+        When forming your description:
+        - **Facts first**: Describe the physical setting, people, and objects exactly as seen.
+        - **Then context**: Briefly note plausible purposes or activities (e.g., “appears to be delivering a package” if carrying a box to a door).
+        - Clearly separate certain facts (“A person is holding a ladder”) from reasonable inferences (“likely performing maintenance”).
+        - Do not speculate beyond what is visible, and do not imply hostility, criminal intent, or other strong judgments unless there is unambiguous visual evidence.
+
+        Here is information already known:
+        - Activity occurred at {review_data["timestamp"].strftime("%I:%M %p")}
+        - Detected objects: {review_data["objects"]}
+        - Recognized objects: {review_data["recognized_objects"]}
+        - Zones involved: {review_data["zones"]}
+
+        Your response **MUST** be a flat JSON object with:
+        - `scene` (string): A full description including setting, entities, actions, and any plausible supported inferences.
+        - `confidence` (float): A number 0–1 for overall confidence in the analysis.
+        - `potential_threat_level` (integer, optional): Include only if there is a clear, observable security concern:
+            - 0 = Normal activity is occurring
+            - 1 = Unusual but not overtly threatening
+            - 2 = Suspicious or potentially harmful
+            - 3 = Clear and immediate threat
+
+        **IMPORTANT:**
+        - Values must be plain strings, floats, or integers — no nested objects, no extra commentary.
+        """
+        logger.debug(
+            f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
+        )
+        response = self._send(context_prompt, thumbnails)
+
+        if response:
+            clean_json = re.sub(
+                r"\n?```$", "", re.sub(r"^```[a-zA-Z0-9]*\n?", "", response)
+            )
+
+            try:
+                return ReviewMetadata.model_validate_json(clean_json)
+            except Exception as e:
+                # rarely LLMs can fail to follow directions on output format
+                logger.warning(
+                    f"Failed to parse review description as the response did not match expected format. {e}"
+                )
+                return None
+        else:
+            return None
+
+    def generate_object_description(
         self,
         camera_config: CameraConfig,
         thumbnails: list[bytes],

frigate/genai/ollama.py

@@ -48,6 +48,7 @@ class OllamaClient(GenAIClient):
                 self.genai_config.model,
                 prompt,
                 images=images,
+                options={"keep_alive": "1h"},
             )
             return result["response"].strip()
         except (TimeoutException, ResponseError) as e:

frigate/review/maintainer.py

@@ -142,6 +142,7 @@ class PendingReviewSegment:
                     "zones": self.zones,
                     "audio": list(self.audio),
                     "thumb_time": self.thumb_time,
+                    "metadata": None,
                 },
             }
         )

frigate/stats/util.py

@@ -356,6 +356,14 @@ def stats_snapshot(
                     embeddings_metrics.yolov9_lpr_pps.value, 2
                 )
 
+        if embeddings_metrics.review_desc_speed.value > 0.0:
+            stats["embeddings"]["review_description_speed"] = round(
+                embeddings_metrics.review_desc_speed.value * 1000, 2
+            )
+            stats["embeddings"]["review_descriptions"] = round(
+                embeddings_metrics.review_desc_dps.value, 2
+            )
+
         for key in embeddings_metrics.classification_speeds.keys():
             stats["embeddings"][f"{key}_classification_speed"] = round(
                 embeddings_metrics.classification_speeds[key].value * 1000, 2

web/public/locales/en/views/settings.json

@@ -150,9 +150,13 @@
       "title": "Streams",
       "desc": "Temporarily disable a camera until Frigate restarts. Disabling a camera completely stops Frigate's processing of this camera's streams. Detection, recording, and debugging will be unavailable.<br /> <em>Note: This does not disable go2rtc restreams.</em>"
     },
-    "genai": {
-      "title": "Generative AI",
-      "desc": "Temporarily enable/disable Generative AI for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    "object_descriptions": {
+      "title": "Generative AI Object Descriptions",
+      "desc": "Temporarily enable/disable Generative AI object descriptions for this camera. When disabled, AI generated descriptions will not be requested for tracked objects on this camera."
+    },
+    "review_descriptions": {
+      "title": "Generative AI Review Descriptions",
+      "desc": "Temporarily enable/disable Generative AI review descriptions for this camera. When disabled, AI generated descriptions will not be requested for review items on this camera."
     },
     "review": {
       "title": "Review",

web/src/api/ws.tsx

@@ -68,7 +68,8 @@ function useValue(): useValueReturn {
         autotracking,
         alerts,
         detections,
-        genai,
+        object_descriptions,
+        review_descriptions,
       } = state["config"];
       cameraStates[`${name}/recordings/state`] = record ? "ON" : "OFF";
       cameraStates[`${name}/enabled/state`] = enabled ? "ON" : "OFF";
@@ -90,7 +91,12 @@ function useValue(): useValueReturn {
       cameraStates[`${name}/review_detections/state`] = detections
         ? "ON"
         : "OFF";
-      cameraStates[`${name}/genai/state`] = genai ? "ON" : "OFF";
+      cameraStates[`${name}/object_descriptions/state`] = object_descriptions
+        ? "ON"
+        : "OFF";
+      cameraStates[`${name}/review_descriptions/state`] = review_descriptions
+        ? "ON"
+        : "OFF";
     });
 
     setWsState((prevState) => ({
@@ -278,14 +284,31 @@ export function useDetectionsState(camera: string): {
   return { payload: payload as ToggleableSetting, send };
 }
 
-export function useGenAIState(camera: string): {
+export function useObjectDescriptionState(camera: string): {
   payload: ToggleableSetting;
   send: (payload: ToggleableSetting, retain?: boolean) => void;
 } {
   const {
     value: { payload },
     send,
-  } = useWs(`${camera}/genai/state`, `${camera}/genai/set`);
+  } = useWs(
+    `${camera}/object_descriptions/state`,
+    `${camera}/object_descriptions/set`,
+  );
+  return { payload: payload as ToggleableSetting, send };
+}
+
+export function useReviewDescriptionState(camera: string): {
+  payload: ToggleableSetting;
+  send: (payload: ToggleableSetting, retain?: boolean) => void;
+} {
+  const {
+    value: { payload },
+    send,
+  } = useWs(
+    `${camera}/review_descriptions/state`,
+    `${camera}/review_descriptions/set`,
+  );
   return { payload: payload as ToggleableSetting, send };
 }
 

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -11,7 +11,11 @@ import { FrigateConfig } from "@/types/frigateConfig";
 import { useFormattedTimestamp } from "@/hooks/use-date-utils";
 import { getIconForLabel } from "@/utils/iconUtil";
 import { useApiHost } from "@/api";
-import { ReviewDetailPaneType, ReviewSegment } from "@/types/review";
+import {
+  ReviewDetailPaneType,
+  ReviewSegment,
+  ThreatLevel,
+} from "@/types/review";
 import { Event } from "@/types/event";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { cn } from "@/lib/utils";
@@ -69,6 +73,25 @@ export default function ReviewDetailDialog({
     review ? ["event_ids", { ids: review.data.detections.join(",") }] : null,
   );
 
+  const aiAnalysis = useMemo(() => review?.data?.metadata, [review]);
+
+  const aiThreatLevel = useMemo(() => {
+    if (!aiAnalysis?.potential_threat_level) {
+      return "None";
+    }
+
+    switch (aiAnalysis.potential_threat_level) {
+      case ThreatLevel.UNUSUAL:
+        return "Unusual Activity";
+      case ThreatLevel.SUSPICIOUS:
+        return "Suspicious Activity";
+      case ThreatLevel.DANGER:
+        return "Danger";
+    }
+
+    return "Unknown";
+  }, [aiAnalysis]);
+
   const hasMismatch = useMemo(() => {
     if (!review || !events) {
       return false;
@@ -232,6 +255,22 @@ export default function ReviewDetailDialog({
           )}
           {pane == "overview" && (
             <div className="flex flex-col gap-5 md:mt-3">
+              {aiAnalysis != undefined && (
+                <div
+                  className={cn(
+                    "m-2 flex h-full w-full flex-col gap-2 rounded-md bg-card p-2",
+                    isDesktop && "w-[90%]",
+                  )}
+                >
+                  AI Analysis
+                  <div className="text-sm text-primary/40">Description</div>
+                  <div className="text-sm">{aiAnalysis.scene}</div>
+                  <div className="text-sm text-primary/40">Score</div>
+                  <div className="text-sm">{aiAnalysis.confidence * 100}%</div>
+                  <div className="text-sm text-primary/40">Threat Level</div>
+                  <div className="text-sm">{aiThreatLevel}</div>
+                </div>
+              )}
               <div className="flex w-full flex-row">
                 <div className="flex w-full flex-col gap-3">
                   <div className="flex flex-col gap-1.5">

web/src/types/frigateConfig.ts

@@ -218,6 +218,12 @@ export interface CameraConfig {
         mode: string;
       };
     };
+    genai?: {
+      enabled: boolean;
+      enabled_in_config: boolean;
+      alerts: boolean;
+      detections: boolean;
+    };
   };
   rtmp: {
     enabled: boolean;

web/src/types/graph.ts

@@ -23,6 +23,11 @@ export const EmbeddingThreshold = {
   error: 1000,
 } as Threshold;
 
+export const GenAIThreshold = {
+  warning: 30000,
+  error: 60000,
+} as Threshold;
+
 export const DetectorTempThreshold = {
   warning: 72,
   error: 80,

web/src/types/review.ts

@@ -18,6 +18,11 @@ export type ReviewData = {
   sub_labels?: string[];
   significant_motion_areas: number[];
   zones: string[];
+  metadata?: {
+    scene: string;
+    confidence: number;
+    potential_threat_level?: number;
+  };
 };
 
 export type SegmentedReviewData =
@@ -73,3 +78,9 @@ export type ConsolidatedSegmentData = {
 };
 
 export type TimelineZoomDirection = "in" | "out" | null;
+
+export enum ThreatLevel {
+  UNUSUAL = 1,
+  SUSPICIOUS = 2,
+  DANGER = 3,
+}

web/src/types/ws.ts

@@ -64,7 +64,8 @@ export interface FrigateCameraState {
     autotracking: boolean;
     alerts: boolean;
     detections: boolean;
-    genai: boolean;
+    object_descriptions: boolean;
+    review_descriptions: boolean;
   };
   motion: boolean;
   objects: ObjectType[];

web/src/views/settings/CameraSettingsView.tsx

@@ -35,7 +35,8 @@ import {
   useAlertsState,
   useDetectionsState,
   useEnabledState,
-  useGenAIState,
+  useObjectDescriptionState,
+  useReviewDescriptionState,
 } from "@/api/ws";
 import CameraEditForm from "@/components/settings/CameraEditForm";
 import { LuPlus } from "react-icons/lu";
@@ -150,8 +151,10 @@ export default function CameraSettingsView({
   const { payload: detectionsState, send: sendDetections } =
     useDetectionsState(selectedCamera);
 
-  const { payload: genAIState, send: sendGenAI } =
-    useGenAIState(selectedCamera);
+  const { payload: objDescState, send: sendObjDesc } =
+    useObjectDescriptionState(selectedCamera);
+  const { payload: revDescState, send: sendRevDesc } =
+    useReviewDescriptionState(selectedCamera);
 
   const handleCheckedChange = useCallback(
     (isChecked: boolean) => {
@@ -418,7 +421,9 @@ export default function CameraSettingsView({
                   <Separator className="my-2 flex bg-secondary" />
 
                   <Heading as="h4" className="my-2">
-                    <Trans ns="views/settings">camera.genai.title</Trans>
+                    <Trans ns="views/settings">
+                      camera.object_descriptions.title
+                    </Trans>
                   </Heading>
 
                   <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
@@ -426,9 +431,9 @@ export default function CameraSettingsView({
                       <Switch
                         id="alerts-enabled"
                         className="mr-3"
-                        checked={genAIState == "ON"}
+                        checked={objDescState == "ON"}
                         onCheckedChange={(isChecked) => {
-                          sendGenAI(isChecked ? "ON" : "OFF");
+                          sendObjDesc(isChecked ? "ON" : "OFF");
                         }}
                       />
                       <div className="space-y-0.5">
@@ -438,7 +443,44 @@ export default function CameraSettingsView({
                       </div>
                     </div>
                     <div className="mt-3 text-sm text-muted-foreground">
-                      <Trans ns="views/settings">camera.genai.desc</Trans>
+                      <Trans ns="views/settings">
+                        camera.object_descriptions.desc
+                      </Trans>
+                    </div>
+                  </div>
+                </>
+              )}
+
+              {cameraConfig?.review?.genai?.enabled_in_config && (
+                <>
+                  <Separator className="my-2 flex bg-secondary" />
+
+                  <Heading as="h4" className="my-2">
+                    <Trans ns="views/settings">
+                      camera.review_descriptions.title
+                    </Trans>
+                  </Heading>
+
+                  <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 space-y-3 text-sm text-primary-variant">
+                    <div className="flex flex-row items-center">
+                      <Switch
+                        id="alerts-enabled"
+                        className="mr-3"
+                        checked={revDescState == "ON"}
+                        onCheckedChange={(isChecked) => {
+                          sendRevDesc(isChecked ? "ON" : "OFF");
+                        }}
+                      />
+                      <div className="space-y-0.5">
+                        <Label htmlFor="genai-enabled">
+                          <Trans>button.enabled</Trans>
+                        </Label>
+                      </div>
+                    </div>
+                    <div className="mt-3 text-sm text-muted-foreground">
+                      <Trans ns="views/settings">
+                        camera.review_descriptions.desc
+                      </Trans>
                     </div>
                   </div>
                 </>

web/src/views/system/EnrichmentMetrics.tsx

@@ -1,8 +1,8 @@
 import useSWR from "swr";
 import { FrigateStats } from "@/types/stats";
-import { useEffect, useMemo, useState } from "react";
+import { useCallback, useEffect, useMemo, useState } from "react";
 import { useFrigateStats } from "@/api/ws";
-import { EmbeddingThreshold } from "@/types/graph";
+import { EmbeddingThreshold, GenAIThreshold, Threshold } from "@/types/graph";
 import { Skeleton } from "@/components/ui/skeleton";
 import { ThresholdBarGraph } from "@/components/graph/SystemGraph";
 import { cn } from "@/lib/utils";
@@ -50,6 +50,14 @@ export default function EnrichmentMetrics({
     }
   }, [initialStats, updatedStats, statsHistory, lastUpdated, setLastUpdated]);
 
+  const getThreshold = useCallback((key: string) => {
+    if (key.includes("description")) {
+      return GenAIThreshold;
+    }
+
+    return EmbeddingThreshold;
+  }, []);
+
   // timestamps
 
   const updateTimes = useMemo(
@@ -65,7 +73,11 @@ export default function EnrichmentMetrics({
     }
 
     const series: {
-      [key: string]: { name: string; data: { x: number; y: number }[] };
+      [key: string]: {
+        name: string;
+        metrics: Threshold;
+        data: { x: number; y: number }[];
+      };
     } = {};
 
     statsHistory.forEach((stats, statsIdx) => {
@@ -79,6 +91,7 @@ export default function EnrichmentMetrics({
         if (!(key in series)) {
           series[key] = {
             name: t("enrichments.embeddings." + rawKey),
+            metrics: getThreshold(rawKey),
             data: [],
           };
         }
@@ -87,7 +100,7 @@ export default function EnrichmentMetrics({
       });
     });
     return Object.values(series);
-  }, [statsHistory, t]);
+  }, [statsHistory, t, getThreshold]);
 
   return (
     <>
@@ -112,7 +125,7 @@ export default function EnrichmentMetrics({
                       graphId={`${series.name}-inference`}
                       name={series.name}
                       unit="ms"
-                      threshold={EmbeddingThreshold}
+                      threshold={series.metrics}
                       updateTimes={updateTimes}
                       data={[series]}
                     />