diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run index 8f5b1c2673..d3aa34236d 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run @@ -55,7 +55,7 @@ function setup_homekit_config() { if [[ ! -f "${config_path}" ]]; then echo "[INFO] Creating empty HomeKit config file..." - echo '{}' > "${config_path}" + echo 'homekit: {}' > "${config_path}" fi # Convert YAML to JSON for jq processing @@ -70,12 +70,14 @@ function setup_homekit_config() { jq ' # Keep only the homekit section if it exists, otherwise empty object if has("homekit") then {homekit: .homekit} else {homekit: {}} end - ' "${temp_json}" > "${cleaned_json}" 2>/dev/null || echo '{"homekit": {}}' > "${cleaned_json}" + ' "${temp_json}" > "${cleaned_json}" 2>/dev/null || { + echo '{"homekit": {}}' > "${cleaned_json}" + } # Convert back to YAML and write to the config file yq eval -P "${cleaned_json}" > "${config_path}" 2>/dev/null || { echo "[WARNING] Failed to convert cleaned config to YAML, creating minimal config" - echo '{"homekit": {}}' > "${config_path}" + echo 'homekit: {}' > "${config_path}" } # Clean up temp files diff --git a/docs/docs/configuration/genai/review_summaries.md b/docs/docs/configuration/genai/review_summaries.md index 8a492f4b8f..04a5b5e94b 100644 --- a/docs/docs/configuration/genai/review_summaries.md +++ b/docs/docs/configuration/genai/review_summaries.md @@ -16,12 +16,13 @@ Review summaries provide structured JSON responses that are saved for each revie ``` - `title` (string): A concise, direct title that describes the purpose or overall action (e.g., "Person taking out trash", "Joe walking dog"). - `scene` (string): A narrative description of what happens across the sequence from start to finish, including setting, detected objects, and their observable actions. +- `shortSummary` (string): A brief 2-sentence summary of the scene, suitable for notifications. This is a condensed version of the scene description. - `confidence` (float): 0-1 confidence in the analysis. Higher confidence when objects/actions are clearly visible and context is unambiguous. - `other_concerns` (list): List of user-defined concerns that may need additional investigation. - `potential_threat_level` (integer): 0, 1, or 2 as defined below. ``` -This will show in multiple places in the UI to give additional context about each activity, and allow viewing more details when extra attention is required. Frigate's built in notifications will also automatically show the title and description when the data is available. +This will show in multiple places in the UI to give additional context about each activity, and allow viewing more details when extra attention is required. Frigate's built in notifications will automatically show the title and `shortSummary` when the data is available, while the full `scene` description is available in the UI for detailed review. ### Defining Typical Activity diff --git a/docs/docs/troubleshooting/memory.md b/docs/docs/troubleshooting/memory.md index 338037c7aa..c74729e5f6 100644 --- a/docs/docs/troubleshooting/memory.md +++ b/docs/docs/troubleshooting/memory.md @@ -36,7 +36,6 @@ Frigate processes are named using a module-based naming scheme. Common module na - `frigate.output` - Output processing - `frigate.audio_manager` - Audio processing - `frigate.embeddings` - Embeddings processing -- `frigate.embeddings_manager` - Embeddings manager You can also specify the full process name (including camera-specific identifiers) if you want to profile a specific camera: diff --git a/frigate/comms/webpush.py b/frigate/comms/webpush.py index 32eeb40e84..62cc12c9a8 100644 --- a/frigate/comms/webpush.py +++ b/frigate/comms/webpush.py @@ -388,7 +388,7 @@ class WebPushClient(Communicator): else: title = base_title - message = payload["after"]["data"]["metadata"]["scene"] + message = payload["after"]["data"]["metadata"]["shortSummary"] else: zone_names = payload["after"]["data"]["zones"] formatted_zone_names = [] diff --git a/frigate/config/config.py b/frigate/config/config.py index 6342c13bf0..a9c54976ee 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -28,6 +28,7 @@ from frigate.util.builtin import ( get_ffmpeg_arg_list, ) from frigate.util.config import ( + CURRENT_CONFIG_VERSION, StreamInfoRetriever, convert_area_to_pixels, find_config_file, @@ -76,11 +77,12 @@ logger = logging.getLogger(__name__) yaml = YAML() -DEFAULT_CONFIG = """ +DEFAULT_CONFIG = f""" mqtt: enabled: False -cameras: {} # No cameras defined, UI wizard should be used +cameras: {{}} # No cameras defined, UI wizard should be used +version: {CURRENT_CONFIG_VERSION} """ DEFAULT_DETECTORS = {"cpu": {"type": "cpu"}} @@ -753,8 +755,7 @@ class FrigateConfig(FrigateBaseModel): if new_config and f.tell() == 0: f.write(DEFAULT_CONFIG) logger.info( - "Created default config file, see the getting started docs \ - for configuration https://docs.frigate.video/guides/getting_started" + "Created default config file, see the getting started docs for configuration: https://docs.frigate.video/guides/getting_started" ) f.seek(0) diff --git a/frigate/data_processing/post/object_descriptions.py b/frigate/data_processing/post/object_descriptions.py index 7bd38bfa8c..cdb5f4fc37 100644 --- a/frigate/data_processing/post/object_descriptions.py +++ b/frigate/data_processing/post/object_descriptions.py @@ -86,7 +86,11 @@ class ObjectDescriptionProcessor(PostProcessorApi): and data["id"] not in self.early_request_sent ): if data["has_clip"] and data["has_snapshot"]: - event: Event = Event.get(Event.id == data["id"]) + try: + event: Event = Event.get(Event.id == data["id"]) + except DoesNotExist: + logger.error(f"Event {data['id']} not found") + return if ( not camera_config.objects.genai.objects diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 0b12aa1a00..0a2754468f 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -92,7 +92,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): pixels_per_image = width * height tokens_per_image = pixels_per_image / 1250 - prompt_tokens = 3500 + prompt_tokens = 3800 response_tokens = 300 available_tokens = context_size - prompt_tokens - response_tokens max_frames = int(available_tokens / tokens_per_image) diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index 70fec9b348..44bb09fb02 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -8,6 +8,9 @@ class ReviewMetadata(BaseModel): scene: str = Field( description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence." ) + shortSummary: str = Field( + description="A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail." + ) confidence: float = Field( description="A float between 0 and 1 representing your overall confidence in this analysis." ) diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 56b49ec67d..fcbb41e661 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -139,8 +139,31 @@ class ONNXModelRunner(BaseModelRunner): ModelTypeEnum.dfine.value, ] - def __init__(self, ort: ort.InferenceSession): + @staticmethod + def is_concurrent_model(model_type: str | None) -> bool: + """Check if model requires thread locking for concurrent inference. + + Some models (like JinaV2) share one runner between text and vision embeddings + called from different threads, requiring thread synchronization. + """ + if not model_type: + return False + + # Import here to avoid circular imports + from frigate.embeddings.types import EnrichmentModelTypeEnum + + return model_type == EnrichmentModelTypeEnum.jina_v2.value + + def __init__(self, ort: ort.InferenceSession, model_type: str | None = None): self.ort = ort + self.model_type = model_type + + # Thread lock to prevent concurrent inference (needed for JinaV2 which shares + # one runner between text and vision embeddings called from different threads) + if self.is_concurrent_model(model_type): + self._inference_lock = threading.Lock() + else: + self._inference_lock = None def get_input_names(self) -> list[str]: return [input.name for input in self.ort.get_inputs()] @@ -150,6 +173,10 @@ class ONNXModelRunner(BaseModelRunner): return self.ort.get_inputs()[0].shape[3] def run(self, input: dict[str, Any]) -> Any | None: + if self._inference_lock: + with self._inference_lock: + return self.ort.run(None, input) + return self.ort.run(None, input) @@ -576,5 +603,6 @@ def get_optimized_runner( ), providers=providers, provider_options=options, - ) + ), + model_type=model_type, ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index c74bc2310c..1a0950cbbc 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -633,7 +633,7 @@ class EmbeddingMaintainer(threading.Thread): camera, frame_name, _, _, motion_boxes, _ = data - if not camera or len(motion_boxes) == 0: + if not camera or len(motion_boxes) == 0 or camera not in self.config.cameras: return camera_config = self.config.cameras[camera] diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 5e1a742798..7f0192912a 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -101,6 +101,7 @@ When forming your description: Your response MUST be a flat JSON object with: - `title` (string): A concise, direct title that describes the primary action or event in the sequence, not just what you literally see. Use spatial context when available to make titles more meaningful. When multiple objects/actions are present, prioritize whichever is most prominent or occurs first. Use names from "Objects in Scene" based on what you visually observe. If you see both a name and an unidentified object of the same type but visually observe only one person/object, use ONLY the name. Examples: "Joe walking dog", "Person taking out trash", "Vehicle arriving in driveway", "Joe accessing vehicle", "Person leaving porch for driveway". - `scene` (string): A narrative description of what happens across the sequence from start to finish, in chronological order. Start by describing how the sequence begins, then describe the progression of events. **Describe all significant movements and actions in the order they occur.** For example, if a vehicle arrives and then a person exits, describe both actions sequentially. **Only describe actions you can actually observe happening in the frames provided.** Do not infer or assume actions that aren't visible (e.g., if you see someone walking but never see them sit, don't say they sat down). Include setting, detected objects, and their observable actions. Avoid speculation or filling in assumed behaviors. Your description should align with and support the threat level you assign. +- `shortSummary` (string): A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail. This should be a condensed version of the scene description above. - `confidence` (float): 0-1 confidence in your analysis. Higher confidence when objects/actions are clearly visible and context is unambiguous. Lower confidence when the sequence is unclear, objects are partially obscured, or context is ambiguous. - `potential_threat_level` (integer): 0, 1, or 2 as defined in "Normal Activity Patterns for This Property" above. Your threat level must be consistent with your scene description and the guidance above. {get_concern_prompt()} @@ -192,6 +193,8 @@ Input format: Each event is a JSON object with: - "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "camera", "time", "start_time", "end_time" - "context": array of related events from other cameras that occurred during overlapping time periods +**Note: Use the "scene" field for event descriptions in the report. Ignore any "shortSummary" field if present.** + Report Structure - Use this EXACT format: # Security Summary - {time_range} diff --git a/frigate/output/output.py b/frigate/output/output.py index 674c02b783..a444150007 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -139,9 +139,11 @@ class OutputProcess(FrigateProcess): if CameraConfigUpdateEnum.add in updates: for camera in updates["add"]: jsmpeg_cameras[camera] = JsmpegCamera( - cam_config, self.stop_event, websocket_server + self.config.cameras[camera], self.stop_event, websocket_server + ) + preview_recorders[camera] = PreviewRecorder( + self.config.cameras[camera] ) - preview_recorders[camera] = PreviewRecorder(cam_config) preview_write_times[camera] = 0 if ( diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index e3ae19159c..bd4368ebe2 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -1604,7 +1604,8 @@ function ObjectDetailsTab({ {config?.cameras[search?.camera].audio_transcription.enabled && search?.label == "speech" && - search?.end_time && ( + search?.end_time && + search?.has_clip && (