Add card

frigate/comms/dispatcher.py

@@ -26,6 +26,7 @@ from frigate.const import (
     UPDATE_EMBEDDINGS_REINDEX_PROGRESS,
     UPDATE_EVENT_DESCRIPTION,
     UPDATE_MODEL_STATE,
+    UPDATE_REVIEW_DESCRIPTION,
     UPSERT_REVIEW_SEGMENT,
 )
 from frigate.models import Event, Previews, Recordings, ReviewSegment
@@ -149,6 +150,14 @@ class Dispatcher:
                 ),
             )
 
+        def handle_update_review_description() -> None:
+            final_data = payload["after"]
+            ReviewSegment.insert(final_data).on_conflict(
+                conflict_target=[ReviewSegment.id],
+                update=final_data,
+            ).execute()
+            self.publish("reviews", json.dumps(payload))
+
         def handle_update_model_state() -> None:
             if payload:
                 model = payload["model"]
@@ -232,6 +241,7 @@ class Dispatcher:
             CLEAR_ONGOING_REVIEW_SEGMENTS: handle_clear_ongoing_review_segments,
             UPDATE_CAMERA_ACTIVITY: handle_update_camera_activity,
             UPDATE_EVENT_DESCRIPTION: handle_update_event_description,
+            UPDATE_REVIEW_DESCRIPTION: handle_update_review_description,
             UPDATE_MODEL_STATE: handle_update_model_state,
             UPDATE_EMBEDDINGS_REINDEX_PROGRESS: handle_update_embeddings_reindex_progress,
             UPDATE_BIRDSEYE_LAYOUT: handle_update_birdseye_layout,

frigate/comms/webpush.py

@@ -369,12 +369,20 @@ class WebPushClient(Communicator):
         sorted_objects.update(payload["after"]["data"]["sub_labels"])
 
         title = f"{titlecase(', '.join(sorted_objects).replace('_', ' '))}{' was' if state == 'end' else ''} detected in {titlecase(', '.join(payload['after']['data']['zones']).replace('_', ' '))}"
-        message = f"Detected on {titlecase(camera.replace('_', ' '))}"
         image = f"{payload['after']['thumb_path'].replace('/media/frigate', '')}"
+        ended = state == "end" or state == "genai"
+
+        if state == "genai" and payload["after"]["data"]["metadata"]:
+            message = payload["after"]["data"]["metadata"]["scene"]
+        else:
+            message = f"Detected on {titlecase(camera.replace('_', ' '))}"
+
+        if ended:
+            print(f"sending a message with message {message}")
 
         # if event is ongoing open to live view otherwise open to recordings view
-        direct_url = f"/review?id={reviewId}" if state == "end" else f"/#{camera}"
+        direct_url = f"/review?id={reviewId}" if ended else f"/#{camera}"
-        ttl = 3600 if state == "end" else 0
+        ttl = 3600 if ended else 0
 
         logger.debug(f"Sending push notification for {camera}, review ID {reviewId}")
 

frigate/const.py

@@ -111,6 +111,7 @@ UPSERT_REVIEW_SEGMENT = "upsert_review_segment"
 CLEAR_ONGOING_REVIEW_SEGMENTS = "clear_ongoing_review_segments"
 UPDATE_CAMERA_ACTIVITY = "update_camera_activity"
 UPDATE_EVENT_DESCRIPTION = "update_event_description"
+UPDATE_REVIEW_DESCRIPTION = "update_review_description"
 UPDATE_MODEL_STATE = "update_model_state"
 UPDATE_EMBEDDINGS_REINDEX_PROGRESS = "handle_embeddings_reindex_progress"
 UPDATE_BIRDSEYE_LAYOUT = "update_birdseye_layout"

frigate/data_processing/post/review_descriptions.py

@@ -10,19 +10,29 @@ from pathlib import Path
 
 import cv2
 
+from frigate.comms.inter_process import InterProcessRequestor
 from frigate.config import FrigateConfig
-from frigate.const import CLIPS_DIR
+from frigate.const import CLIPS_DIR, UPDATE_REVIEW_DESCRIPTION
 from frigate.data_processing.types import PostProcessDataEnum
 from frigate.genai import GenAIClient
 
 from ..post.api import PostProcessorApi
+from ..types import DataProcessorMetrics
 
 logger = logging.getLogger(__name__)
 
 
 class ReviewDescriptionProcessor(PostProcessorApi):
-    def __init__(self, config: FrigateConfig, metrics, client: GenAIClient):
+    def __init__(
+        self,
+        config: FrigateConfig,
+        requestor: InterProcessRequestor,
+        metrics: DataProcessorMetrics,
+        client: GenAIClient,
+    ):
         super().__init__(config, metrics, None)
+        self.requestor = requestor
+        self.metrics = metrics
         self.tracked_review_items: dict[str, list[tuple[int, bytes]]] = {}
         self.genai_client = client
 
@@ -92,8 +102,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
 
             # kickoff analysis
             threading.Thread(
-                target=self.run_analysis,
+                target=run_analysis,
                 args=(
+                    self.requestor,
                     self.genai_client,
                     camera,
                     final_data,
@@ -102,23 +113,39 @@ class ReviewDescriptionProcessor(PostProcessorApi):
             ).start()
             self.tracked_review_items.pop(id)
 
-    def run_analysis(
-        self,
-        genai_client: GenAIClient,
-        camera: str,
-        final_data: dict[str, str],
-        thumbs: list[bytes],
-    ) -> None:
-        genai_client.generate_review_description(
-            {
-                "camera": camera,
-                "objects": final_data["data"]["objects"],
-                "recognized_objects": final_data["data"]["sub_labels"],
-                "zones": final_data["data"]["zones"],
-                "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
-            },
-            thumbs,
-        )
-
     def handle_request(self, request_data):
         pass
+
+
+@staticmethod
+def run_analysis(
+    requestor: InterProcessRequestor,
+    genai_client: GenAIClient,
+    camera: str,
+    final_data: dict[str, str],
+    thumbs: list[bytes],
+) -> None:
+    metadata = genai_client.generate_review_description(
+        {
+            "camera": camera,
+            "objects": final_data["data"]["objects"],
+            "recognized_objects": final_data["data"]["sub_labels"],
+            "zones": final_data["data"]["zones"],
+            "timestamp": datetime.datetime.fromtimestamp(final_data["end_time"]),
+        },
+        thumbs,
+    )
+
+    if not metadata:
+        return None
+
+    prev_data = copy.deepcopy(final_data)
+    final_data["data"]["metadata"] = metadata.model_dump()
+    requestor.send_data(
+        UPDATE_REVIEW_DESCRIPTION,
+        {
+            "type": "genai",
+            "before": {k: v for k, v in prev_data.items()},
+            "after": {k: v for k, v in final_data.items()},
+        },
+    )

frigate/embeddings/maintainer.py

@@ -209,7 +209,9 @@ class EmbeddingMaintainer(threading.Thread):
 
         if any(c.review.genai.enabled_in_config for c in self.config.cameras.values()):
             self.post_processors.append(
-                ReviewDescriptionProcessor(self.config, self.metrics, self.genai_client)
+                ReviewDescriptionProcessor(
+                    self.config, self.requestor, self.metrics, self.genai_client
+                )
             )
 
         if self.config.lpr.enabled:

frigate/genai/__init__.py

@@ -3,6 +3,7 @@
 import importlib
 import logging
 import os
+import re
 from typing import Any, Optional
 
 from playhouse.shortcuts import model_to_dict
@@ -36,7 +37,7 @@ class GenAIClient:
 
     def generate_review_description(
         self, review_data: dict[str, Any], thumbnails: list[bytes]
-    ) -> None:
+    ) -> ReviewMetadata | None:
         """Generate a description for the review item activity."""
         context_prompt = f"""
         Please analyze the image(s), which are in chronological order, strictly from the perspective of the {review_data["camera"].replace("_", " ")} security camera.
@@ -64,12 +65,22 @@ class GenAIClient:
         Omit this field entirely if there is no observable security concern.
 
         **IMPORTANT:**
-        - Values for each field must be plain strings or integers — no nested objects or explanatory text.
+        Values for each field must be plain strings or integers — no nested objects or explanatory text.
-        - The JSON must strictly match this structure:
-        {ReviewMetadata.model_json_schema()["properties"]}
         """
-        logger.info(f"processing {review_data}")
+        response = self._send(context_prompt, thumbnails)
-        logger.info(f"Got GenAI review: {self._send(context_prompt, thumbnails)}")
+
+        if response:
+            clean_json = re.sub(
+                r"\n?```$", "", re.sub(r"^```[a-zA-Z0-9]*\n?", "", response)
+            )
+
+            try:
+                return ReviewMetadata.model_validate_json(clean_json)
+            except Exception:
+                # rarely LLMs can fail to follow directions on output format
+                return None
+        else:
+            return None
 
     def generate_object_description(
         self,

frigate/review/maintainer.py

@@ -142,6 +142,7 @@ class PendingReviewSegment:
                     "zones": self.zones,
                     "audio": list(self.audio),
                     "thumb_time": self.thumb_time,
+                    "metadata": None,
                 },
             }
         )

web/src/components/overlay/detail/ReviewDetailDialog.tsx

@@ -69,6 +69,8 @@ export default function ReviewDetailDialog({
     review ? ["event_ids", { ids: review.data.detections.join(",") }] : null,
   );
 
+  const aiAnalysis = useMemo(() => review?.data?.metadata, [review]);
+
   const hasMismatch = useMemo(() => {
     if (!review || !events) {
       return false;
@@ -232,6 +234,15 @@ export default function ReviewDetailDialog({
           )}
           {pane == "overview" && (
             <div className="flex flex-col gap-5 md:mt-3">
+              {aiAnalysis != undefined && (
+                <div className="m-2 flex h-full w-[90%] flex-col gap-2 rounded-md bg-card p-2">
+                  AI Analysis
+                  <div className="text-sm text-primary/40">Description</div>
+                  <div className="text-sm smart-capitalize">
+                    {aiAnalysis.scene}
+                  </div>
+                </div>
+              )}
               <div className="flex w-full flex-row">
                 <div className="flex w-full flex-col gap-3">
                   <div className="flex flex-col gap-1.5">

web/src/types/review.ts

@@ -18,6 +18,9 @@ export type ReviewData = {
   sub_labels?: string[];
   significant_motion_areas: number[];
   zones: string[];
+  metadata?: {
+    scene: string;
+  };
 };
 
 export type SegmentedReviewData =

web/vite.config.ts

@@ -4,7 +4,7 @@ import { defineConfig } from "vite";
 import react from "@vitejs/plugin-react-swc";
 import monacoEditorPlugin from "vite-plugin-monaco-editor";
 
-const proxyHost = process.env.PROXY_HOST || "localhost:5000";
+const proxyHost = process.env.PROXY_HOST || "192.168.50.106:5002";
 
 // https://vitejs.dev/config/
 export default defineConfig({