From 05ef72f3a56423cb51ff330be41b1be8f329367e Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Thu, 11 Dec 2025 07:18:02 -0700 Subject: [PATCH] Improve security summary format and usefulness --- .../post/review_descriptions.py | 44 +++++---- frigate/genai/__init__.py | 96 ++++++++----------- 2 files changed, 64 insertions(+), 76 deletions(-) diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 7965490fa..7932d56f4 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -251,20 +251,22 @@ class ReviewDescriptionProcessor(PostProcessorApi): if not primary_segments: return "No concerns were found during this time period." - # For each primary segment, find overlapping contextual items from other cameras - all_items_for_summary = [] + # Build hierarchical structure: each primary event with its contextual items + events_with_context = [] for primary_seg in primary_segments: - # Add the primary item with marker + # Start building the primary event structure primary_item = copy.deepcopy(primary_seg["metadata"]) - primary_item["_is_primary"] = True - primary_item["_camera"] = primary_seg["camera"] - all_items_for_summary.append(primary_item) + primary_item["camera"] = primary_seg["camera"] + primary_item["start_time"] = primary_seg["start_time"] + primary_item["end_time"] = primary_seg["end_time"] # Find overlapping contextual items from other cameras primary_start = primary_seg["start_time"] primary_end = primary_seg["end_time"] primary_camera = primary_seg["camera"] + contextual_items = [] + seen_contextual_cameras = set() for seg in segments: seg_camera = seg["camera"] @@ -279,21 +281,25 @@ class ReviewDescriptionProcessor(PostProcessorApi): seg_end = seg["end_time"] if seg_start < primary_end and primary_start < seg_end: - contextual_item = copy.deepcopy(seg["metadata"]) - contextual_item["_is_primary"] = False - contextual_item["_camera"] = seg_camera - contextual_item["_related_to_camera"] = primary_camera + # Avoid duplicates if same camera has multiple overlapping segments + if seg_camera not in seen_contextual_cameras: + contextual_item = copy.deepcopy(seg["metadata"]) + contextual_item["camera"] = seg_camera + contextual_item["start_time"] = seg_start + contextual_item["end_time"] = seg_end + contextual_items.append(contextual_item) + seen_contextual_cameras.add(seg_camera) - if not any( - item.get("_camera") == seg_camera - and item.get("time") == contextual_item.get("time") - for item in all_items_for_summary - ): - all_items_for_summary.append(contextual_item) + # Add context array to primary item + primary_item["context"] = contextual_items + events_with_context.append(primary_item) + total_context_items = sum( + len(event.get("context", [])) for event in events_with_context + ) logger.debug( - f"Summary includes {len(primary_segments)} primary items and " - f"{len(all_items_for_summary) - len(primary_segments)} contextual items" + f"Summary includes {len(events_with_context)} primary events with " + f"{total_context_items} total contextual items" ) if self.config.review.genai.debug_save_thumbnails: @@ -304,7 +310,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): return self.genai_client.generate_review_summary( start_ts, end_ts, - all_items_for_summary, + events_with_context, self.config.review.genai.debug_save_thumbnails, ) else: diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 05c8d3926..910fc13b9 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -177,78 +177,60 @@ Each line represents a detection state, not necessarily unique individuals. Pare self, start_ts: float, end_ts: float, - segments: list[dict[str, Any]], + events: list[dict[str, Any]], debug_save: bool, ) -> str | None: """Generate a summary of review item descriptions over a period of time.""" time_range = f"{datetime.datetime.fromtimestamp(start_ts).strftime('%B %d, %Y at %I:%M %p')} to {datetime.datetime.fromtimestamp(end_ts).strftime('%B %d, %Y at %I:%M %p')}" timeline_summary_prompt = f""" -You are a security officer. -Time range: {time_range}. -Input: JSON list with "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "_is_primary", "_camera". +You are a security officer writing a concise security report. -Task: Write a concise, human-presentable security report in markdown format. +Time range: {time_range} -CRITICAL - Understanding Primary vs Contextual Items: -- Items with "_is_primary": true are events that REQUIRE REVIEW and MUST be included in the report -- Items with "_is_primary": false are additional context from other camera perspectives that overlap in time -- **DO NOT create separate bullet points or sections for contextual items** -- **ONLY use contextual items to enrich and inform the description of primary items** -- The "_camera" field indicates which camera captured each event -- **When a contextual item provides relevant background, you MUST incorporate it directly into the primary event's bullet point** -- Contextual information often explains or de-escalates seemingly suspicious primary events +Input format: Each event is a JSON object with: +- "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "camera", "time", "start_time", "end_time" +- "context": array of related events from other cameras that occurred during overlapping time periods -Rules for the report: +Report Structure - Use this EXACT format: -- Title & overview - - Start with: - # Security Summary - {time_range} - - Write a 1-2 sentence situational overview capturing the general pattern of the period. - - Keep the overview high-level; specific details will be in the event bullets below. +# Security Summary - {time_range} -- Event details - - **ONLY create bullet points for PRIMARY items (_is_primary: true)** - - **Do NOT create sections or bullets for events that don't exist** - - Do NOT create separate bullets for contextual items - - Present primary events in chronological order as a bullet list. - - **CRITICAL: When contextual items overlap with a primary event, you MUST weave that information directly into the same bullet point** - - Format: **[Timestamp]** - [Description incorporating any contextual information]. [Camera info]. (threat level: X) - - If contextual information provides an explanation (e.g., delivery truck → person is likely delivery driver), reflect this understanding in your description and potentially adjust the perceived threat level - - If multiple PRIMARY events occur within the same minute, combine them into a single bullet with sub-points. - - Use bold timestamps for clarity. - - Camera format: "Camera: [camera name]" or mention contextual cameras inline when relevant - - Group bullets under subheadings ONLY when you have actual PRIMARY events to list (e.g., Porch Activity, Unusual Behavior). +## Overview +[Write 1-2 sentences summarizing the overall activity pattern during this period.] -- Threat levels - - Show the threat level for PRIMARY events using these labels: - - Threat level 0: "Normal" - - Threat level 1: "Needs review" - - Threat level 2: "Security concern" - - Format as (threat level: Normal), (threat level: Needs review), or (threat level: Security concern). - - **When contextual items clearly explain a primary event (e.g., delivery truck explains person at door), you should describe it as normal activity and note the explanation** - - **Your description and tone should reflect the fuller understanding provided by contextual information** - - Example: Primary event says "unidentified person with face covering" but context shows delivery truck → describe as "delivery person (truck visible on Front Driveway Cam)" rather than emphasizing suspicious elements - - The stored threat level remains as originally classified, but your narrative should reflect the contextual understanding - - If multiple PRIMARY events at the same time share the same threat level, only state it once. +--- -- Final assessment - - End with a Final Assessment section. - - If all primary events are threat level 0 or explained by contextual items: - Final assessment: Only normal residential activity observed during this period. - - If threat level 1 events are present: - Final assessment: Some activity requires review but no security concerns identified. - - If threat level 2 events are present, clearly summarize them as Security concerns requiring immediate attention. - - Keep this section brief - do not repeat details from the event descriptions above. +## Timeline -- Conciseness - - Do not repeat benign clothing/appearance details unless they distinguish individuals. - - Summarize similar routine events instead of restating full scene descriptions. - - When incorporating contextual information, do so briefly and naturally within the primary event description. - - Avoid lengthy explanatory notes - integrate context seamlessly into the narrative. +[Group events by time periods (e.g., "Morning (6:00 AM - 12:00 PM)", "Afternoon (12:00 PM - 5:00 PM)", "Evening (5:00 PM - 9:00 PM)", "Night (9:00 PM - 6:00 AM)"). Use appropriate time blocks based on when events occurred.] + +### [Time Block Name] + +**HH:MM AM/PM** | [Camera Name] | [Threat Level Indicator] +- [Event title]: [Clear description incorporating contextual information from the "context" array] +- Context: [If context array has items, mention them here, e.g., "Delivery truck present on Front Driveway Cam (HH:MM AM/PM)"] +- Assessment: [Brief assessment incorporating context - if context explains the event, note it here] + +[Repeat for each event in chronological order within the time block] + +--- + +## Summary +[One sentence summarizing the period. If all events are normal/explained: "Routine activity observed." If review needed: "Some activity requires review but no security concerns." If security concerns: "Security concerns requiring immediate attention."] + +Guidelines: +- List ALL events in chronological order, grouped by time blocks +- Threat level indicators: ✓ Normal, ⚠️ Needs review, 🔴 Security concern +- Integrate contextual information naturally - use the "context" array to enrich each event's description +- If context explains the event (e.g., delivery truck explains person at door), describe it accordingly (e.g., "delivery person" not "unidentified person") +- Be concise but informative - focus on what happened and what it means +- If contextual information makes an event clearly normal, reflect that in your assessment +- Only create time blocks that have events - don't create empty sections """ - for item in segments: - timeline_summary_prompt += f"\n{item}" + timeline_summary_prompt += "\n\nEvents:\n" + for event in events: + timeline_summary_prompt += f"\n{event}\n" if debug_save: with open(