Migrate object genai configuration (#19437)

* Move genAI object to objects section

* Adjust config propogation behavior

* Refactor genai config usage

* Automatic migration

* Always start the embeddings process

* Always init embeddings

* Config fixes

* Adjust reference config

* Adjust docs

* Formatting

* Fix

.github/workflows/pull_request.yml

@@ -107,7 +107,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build
-        run: make
+        run: make debug
       - name: Run mypy
         run: docker run --rm --entrypoint=python3 frigate:latest -u -m mypy --config-file frigate/mypy.ini frigate
       - name: Run tests

Makefile

@@ -20,6 +20,12 @@ local: version
 		--tag frigate:latest \
 		--load
 
+debug: version
+	docker buildx build --target=frigate --file docker/main/Dockerfile . \
+	    --build-arg DEBUG=true \
+		--tag frigate:latest \
+		--load
+
 amd64:
 	docker buildx build --target=frigate --file docker/main/Dockerfile . \
 		--tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH) \

docker/main/Dockerfile

@@ -148,6 +148,7 @@ RUN --mount=type=bind,source=docker/main/install_s6_overlay.sh,target=/deps/inst
 FROM base AS wheels
 ARG DEBIAN_FRONTEND
 ARG TARGETARCH
+ARG DEBUG=false
 
 # Use a separate container to build wheels to prevent build dependencies in final image
 RUN apt-get -qq update \
@@ -177,6 +178,8 @@ RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
     && python3 get-pip.py "pip"
 
 COPY docker/main/requirements.txt /requirements.txt
+COPY docker/main/requirements-dev.txt /requirements-dev.txt
+
 RUN pip3 install -r /requirements.txt
 
 # Build pysqlite3 from source
@@ -184,7 +187,10 @@ COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh
 RUN /build_pysqlite3.sh
 
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
-RUN pip3 wheel --wheel-dir=/wheels -r /requirements-wheels.txt
+RUN pip3 wheel --wheel-dir=/wheels -r /requirements-wheels.txt && \
+    if [ "$DEBUG" = "true" ]; then \
+        pip3 wheel --wheel-dir=/wheels -r /requirements-dev.txt; \
+    fi
 
 # Install HailoRT & Wheels
 RUN --mount=type=bind,source=docker/main/install_hailort.sh,target=/deps/install_hailort.sh \

docker/main/requirements-dev.txt

@@ -1 +1,4 @@
 ruff
+
+# types
+types-peewee == 3.17.*

docs/docs/configuration/genai.md

@@ -15,13 +15,13 @@ To use Generative AI, you must define a single provider at the global level of y
 
 ```yaml
 genai:
-  enabled: True
   provider: gemini
   api_key: "{FRIGATE_GEMINI_API_KEY}"
   model: gemini-1.5-flash
 
 cameras:
   front_camera:
+    objects:
     genai:
         enabled: True # <- enable GenAI for your front camera
         use_snapshot: True
@@ -30,6 +30,7 @@ cameras:
         required_zones:
           - steps
   indoor_camera:
+    objects:
       genai:
         enabled: False # <- disable GenAI for your indoor camera
 ```
@@ -68,7 +69,6 @@ You should have at least 8 GB of RAM available (or VRAM if running on GPU) to ru
 
 ```yaml
 genai:
-  enabled: True
   provider: ollama
   base_url: http://localhost:11434
   model: llava:7b
@@ -95,7 +95,6 @@ To start using Gemini, you must first get an API key from [Google AI Studio](htt
 
 ```yaml
 genai:
-  enabled: True
   provider: gemini
   api_key: "{FRIGATE_GEMINI_API_KEY}"
   model: gemini-1.5-flash
@@ -117,7 +116,6 @@ To start using OpenAI, you must first [create an API key](https://platform.opena
 
 ```yaml
 genai:
-  enabled: True
   provider: openai
   api_key: "{FRIGATE_OPENAI_API_KEY}"
   model: gpt-4o
@@ -145,7 +143,6 @@ To start using Azure OpenAI, you must first [create a resource](https://learn.mi
 
 ```yaml
 genai:
-  enabled: True
   provider: azure_openai
   base_url: https://example-endpoint.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2023-03-15-preview
   api_key: "{FRIGATE_OPENAI_API_KEY}"
@@ -188,22 +185,25 @@ You are also able to define custom prompts in your configuration.
 
 ```yaml
 genai:
-  enabled: True
   provider: ollama
   base_url: http://localhost:11434
   model: llava
+
+objects:
     prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
     object_prompts:
       person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
       car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
 ```
 
-Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
+Prompts can also be overridden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
 
 ```yaml
 cameras:
   front_door:
+    objects:
       genai:
+        enabled: True
         use_snapshot: True
         prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
         object_prompts:

docs/docs/configuration/reference.md

@@ -339,6 +339,33 @@ objects:
       # Optional: mask to prevent this object type from being detected in certain areas (default: no mask)
       # Checks based on the bottom center of the bounding box of the object
       mask: 0.000,0.000,0.781,0.000,0.781,0.278,0.000,0.278
+  # Optional: Configuration for AI generated tracked object descriptions
+  genai:
+    # Optional: Enable AI object description generation (default: shown below)
+    enabled: False
+    # Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
+    use_snapshot: False
+    # Optional: The default prompt for generating descriptions. Can use replacement
+    # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
+    prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
+    # Optional: Object specific prompts to customize description results
+    # Format: {label}: {prompt}
+    object_prompts:
+      person: "My special person prompt."
+    # Optional: objects to generate descriptions for (default: all objects that are tracked)
+    objects:
+      - person
+      - cat
+    # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
+    required_zones: []
+    # Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
+    send_triggers:
+      # Once the object is no longer tracked
+      tracked_object_end: True
+      # Optional: After X many significant updates are received (default: shown below)
+      after_significant_updates: None
+    # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
+    debug_save_thumbnails: False
 
 # Optional: Review configuration
 # NOTE: Can be overridden at the camera level
@@ -612,13 +639,6 @@ genai:
   base_url: http://localhost::11434
   # Required if gemini or openai
   api_key: "{FRIGATE_GENAI_API_KEY}"
-  # Optional: The default prompt for generating descriptions. Can use replacement
-  # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
-  prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
-  # Optional: Object specific prompts to customize description results
-  # Format: {label}: {prompt}
-  object_prompts:
-    person: "My special person prompt."
 
 # Optional: Configuration for audio transcription
 # NOTE: only the enabled option can be overridden at the camera level
@@ -857,34 +877,6 @@ cameras:
         actions:
           - notification
 
-    # Optional: Configuration for AI generated tracked object descriptions
-    genai:
-      # Optional: Enable AI description generation (default: shown below)
-      enabled: False
-      # Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
-      use_snapshot: False
-      # Optional: The default prompt for generating descriptions. Can use replacement
-      # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
-      prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
-      # Optional: Object specific prompts to customize description results
-      # Format: {label}: {prompt}
-      object_prompts:
-        person: "My special person prompt."
-      # Optional: objects to generate descriptions for (default: all objects that are tracked)
-      objects:
-        - person
-        - cat
-      # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
-      required_zones: []
-      # Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
-      send_triggers:
-        # Once the object is no longer tracked
-        tracked_object_end: True
-        # Optional: After X many significant updates are received (default: shown below)
-        after_significant_updates: None
-      # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
-      debug_save_thumbnails: False
-
 # Optional
 ui:
   # Optional: Set a timezone to use in the UI (default: use browser local time)

frigate/api/event.py

@@ -1230,7 +1230,7 @@ def regenerate_description(
 
     camera_config = request.app.frigate_config.cameras[event.camera]
 
-    if camera_config.genai.enabled or params.force:
+    if camera_config.objects.genai.enabled or params.force:
         request.app.event_metadata_updater.publish(
             (event.id, params.source, params.force),
             EventMetadataTypeEnum.regenerate_description.value,

frigate/app.py

@@ -246,18 +246,7 @@ class FrigateApp:
         logger.info(f"Review process started: {review_segment_process.pid}")
 
     def init_embeddings_manager(self) -> None:
-        genai_cameras = [
+        # always start the embeddings process
-            c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
-        ]
-
-        if (
-            not self.config.semantic_search.enabled
-            and not genai_cameras
-            and not self.config.lpr.enabled
-            and not self.config.face_recognition.enabled
-        ):
-            return
-
         embedding_process = EmbeddingProcess(
             self.config, self.embeddings_metrics, self.stop_event
         )
@@ -309,18 +298,6 @@ class FrigateApp:
             migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
 
     def init_embeddings_client(self) -> None:
-        genai_cameras = [
-            c
-            for c in self.config.cameras.values()
-            if c.enabled_in_config and c.genai.enabled
-        ]
-
-        if (
-            self.config.semantic_search.enabled
-            or self.config.lpr.enabled
-            or genai_cameras
-            or self.config.face_recognition.enabled
-        ):
         # Create a client for other processes to use
         self.embeddings = EmbeddingsContext(self.db)
 

frigate/comms/dispatcher.py

@@ -3,7 +3,7 @@
 import datetime
 import json
 import logging
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, cast
 
 from frigate.camera import PTZMetrics
 from frigate.camera.activity_manager import CameraActivityManager
@@ -135,7 +135,7 @@ class Dispatcher:
 
         def handle_update_event_description() -> None:
             event: Event = Event.get(Event.id == payload["id"])
-            event.data["description"] = payload["description"]
+            cast(dict, event.data)["description"] = payload["description"]
             event.save()
             self.publish(
                 "tracked_object_update",
@@ -209,7 +209,7 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
-                    "genai": self.config.cameras[camera].genai.enabled,
+                    "genai": self.config.cameras[camera].objects.genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -744,10 +744,10 @@ class Dispatcher:
 
     def _on_genai_command(self, camera_name: str, payload: str) -> None:
         """Callback for GenAI topic."""
-        genai_settings = self.config.cameras[camera_name].genai
+        genai_settings = self.config.cameras[camera_name].objects.genai
 
         if payload == "ON":
-            if not self.config.cameras[camera_name].genai.enabled_in_config:
+            if not self.config.cameras[camera_name].objects.genai.enabled_in_config:
                 logger.error(
                     "GenAI must be enabled in the config to be turned on via MQTT."
                 )

frigate/comms/mqtt.py

@@ -124,7 +124,7 @@ class MqttClient(Communicator):
             )
             self.publish(
                 f"{camera_name}/genai/state",
-                "ON" if camera.genai.enabled_in_config else "OFF",
+                "ON" if camera.objects.genai.enabled_in_config else "OFF",
                 retain=True,
             )
 

frigate/comms/webpush.py

@@ -70,7 +70,7 @@ class WebPushClient(Communicator):
         # Pull keys from PEM or generate if they do not exist
         self.vapid = Vapid01.from_file(os.path.join(CONFIG_DIR, "notifications.pem"))
 
-        users: list[User] = (
+        users: list[dict[str, Any]] = (
             User.select(User.username, User.notification_tokens).dicts().iterator()
         )
         for user in users:

frigate/config/camera/camera.py

@@ -28,7 +28,6 @@ from .audio import AudioConfig
 from .birdseye import BirdseyeCameraConfig
 from .detect import DetectConfig
 from .ffmpeg import CameraFfmpegConfig, CameraInput
-from .genai import GenAICameraConfig
 from .live import CameraLiveConfig
 from .motion import MotionConfig
 from .mqtt import CameraMqttConfig
@@ -71,9 +70,6 @@ class CameraConfig(FrigateBaseModel):
         default_factory=CameraFaceRecognitionConfig, title="Face recognition config."
     )
     ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.")
-    genai: GenAICameraConfig = Field(
-        default_factory=GenAICameraConfig, title="Generative AI configuration."
-    )
     live: CameraLiveConfig = Field(
         default_factory=CameraLiveConfig, title="Live playback settings."
     )

frigate/config/camera/genai.py

@@ -1,12 +1,12 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import Optional
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import Field
 
 from ..base import FrigateBaseModel
 from ..env import EnvString
 
-__all__ = ["GenAIConfig", "GenAICameraConfig", "GenAIProviderEnum"]
+__all__ = ["GenAIConfig", "GenAIProviderEnum"]
 
 
 class GenAIProviderEnum(str, Enum):
@@ -16,70 +16,8 @@ class GenAIProviderEnum(str, Enum):
     ollama = "ollama"
 
 
-class GenAISendTriggersConfig(BaseModel):
-    tracked_object_end: bool = Field(
-        default=True, title="Send once the object is no longer tracked."
-    )
-    after_significant_updates: Optional[int] = Field(
-        default=None,
-        title="Send an early request to generative AI when X frames accumulated.",
-        ge=1,
-    )
-
-
-# uses BaseModel because some global attributes are not available at the camera level
-class GenAICameraConfig(BaseModel):
-    enabled: bool = Field(default=False, title="Enable GenAI for camera.")
-    use_snapshot: bool = Field(
-        default=False, title="Use snapshots for generating descriptions."
-    )
-    prompt: str = Field(
-        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
-        title="Default caption prompt.",
-    )
-    object_prompts: dict[str, str] = Field(
-        default_factory=dict, title="Object specific prompts."
-    )
-
-    objects: Union[str, list[str]] = Field(
-        default_factory=list,
-        title="List of objects to run generative AI for.",
-    )
-    required_zones: Union[str, list[str]] = Field(
-        default_factory=list,
-        title="List of required zones to be entered in order to run generative AI.",
-    )
-    debug_save_thumbnails: bool = Field(
-        default=False,
-        title="Save thumbnails sent to generative AI for debugging purposes.",
-    )
-    send_triggers: GenAISendTriggersConfig = Field(
-        default_factory=GenAISendTriggersConfig,
-        title="What triggers to use to send frames to generative AI for a tracked object.",
-    )
-
-    enabled_in_config: Optional[bool] = Field(
-        default=None, title="Keep track of original state of generative AI."
-    )
-
-    @field_validator("required_zones", mode="before")
-    @classmethod
-    def validate_required_zones(cls, v):
-        if isinstance(v, str) and "," not in v:
-            return [v]
-
-        return v
-
-
 class GenAIConfig(FrigateBaseModel):
-    enabled: bool = Field(default=False, title="Enable GenAI.")
+    """Primary GenAI Config to define GenAI Provider."""
-    prompt: str = Field(
-        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
-        title="Default caption prompt.",
-    )
-    object_prompts: dict[str, str] = Field(
-        default_factory=dict, title="Object specific prompts."
-    )
 
     api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
     base_url: Optional[str] = Field(default=None, title="Provider base url.")

frigate/config/camera/objects.py

@@ -1,10 +1,10 @@
 from typing import Any, Optional, Union
 
-from pydantic import Field, PrivateAttr, field_serializer
+from pydantic import Field, PrivateAttr, field_serializer, field_validator
 
 from ..base import FrigateBaseModel
 
-__all__ = ["ObjectConfig", "FilterConfig"]
+__all__ = ["ObjectConfig", "GenAIObjectConfig", "FilterConfig"]
 
 
 DEFAULT_TRACKED_OBJECTS = ["person"]
@@ -49,12 +49,69 @@ class FilterConfig(FrigateBaseModel):
         return None
 
 
+class GenAIObjectTriggerConfig(FrigateBaseModel):
+    tracked_object_end: bool = Field(
+        default=True, title="Send once the object is no longer tracked."
+    )
+    after_significant_updates: Optional[int] = Field(
+        default=None,
+        title="Send an early request to generative AI when X frames accumulated.",
+        ge=1,
+    )
+
+
+class GenAIObjectConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable GenAI for camera.")
+    use_snapshot: bool = Field(
+        default=False, title="Use snapshots for generating descriptions."
+    )
+    prompt: str = Field(
+        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
+        title="Default caption prompt.",
+    )
+    object_prompts: dict[str, str] = Field(
+        default_factory=dict, title="Object specific prompts."
+    )
+
+    objects: Union[str, list[str]] = Field(
+        default_factory=list,
+        title="List of objects to run generative AI for.",
+    )
+    required_zones: Union[str, list[str]] = Field(
+        default_factory=list,
+        title="List of required zones to be entered in order to run generative AI.",
+    )
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
+    send_triggers: GenAIObjectTriggerConfig = Field(
+        default_factory=GenAIObjectTriggerConfig,
+        title="What triggers to use to send frames to generative AI for a tracked object.",
+    )
+    enabled_in_config: Optional[bool] = Field(
+        default=None, title="Keep track of original state of generative AI."
+    )
+
+    @field_validator("required_zones", mode="before")
+    @classmethod
+    def validate_required_zones(cls, v):
+        if isinstance(v, str) and "," not in v:
+            return [v]
+
+        return v
+
+
 class ObjectConfig(FrigateBaseModel):
     track: list[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.")
     filters: dict[str, FilterConfig] = Field(
         default_factory=dict, title="Object filters."
     )
     mask: Union[str, list[str]] = Field(default="", title="Object mask.")
+    genai: GenAIObjectConfig = Field(
+        default_factory=GenAIObjectConfig,
+        title="Config for using genai to analyze objects.",
+    )
     _all_objects: list[str] = PrivateAttr()
 
     @property

frigate/config/camera/updater.py

@@ -99,7 +99,7 @@ class CameraConfigUpdateSubscriber:
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
         elif update_type == CameraConfigUpdateEnum.genai:
-            config.genai = updated_config
+            config.objects.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
         elif update_type == CameraConfigUpdateEnum.notifications:

frigate/config/config.py

@@ -352,6 +352,11 @@ class FrigateConfig(FrigateBaseModel):
         default_factory=ModelConfig, title="Detection model configuration."
     )
 
+    # GenAI config
+    genai: GenAIConfig = Field(
+        default_factory=GenAIConfig, title="Generative AI configuration."
+    )
+
     # Camera config
     cameras: Dict[str, CameraConfig] = Field(title="Camera configuration.")
     audio: AudioConfig = Field(
@@ -366,9 +371,6 @@ class FrigateConfig(FrigateBaseModel):
     ffmpeg: FfmpegConfig = Field(
         default_factory=FfmpegConfig, title="Global FFmpeg configuration."
     )
-    genai: GenAIConfig = Field(
-        default_factory=GenAIConfig, title="Generative AI configuration."
-    )
     live: CameraLiveConfig = Field(
         default_factory=CameraLiveConfig, title="Live playback settings."
     )
@@ -458,7 +460,6 @@ class FrigateConfig(FrigateBaseModel):
                 "live": ...,
                 "objects": ...,
                 "review": ...,
-                "genai": ...,
                 "motion": ...,
                 "notifications": ...,
                 "detect": ...,
@@ -606,7 +607,9 @@ class FrigateConfig(FrigateBaseModel):
             camera_config.review.detections.enabled_in_config = (
                 camera_config.review.detections.enabled
             )
-            camera_config.genai.enabled_in_config = camera_config.genai.enabled
+            camera_config.objects.genai.enabled_in_config = (
+                camera_config.objects.genai.enabled
+            )
 
             # Add default filters
             object_keys = camera_config.objects.track

frigate/embeddings/maintainer.py

@@ -30,7 +30,7 @@ from frigate.comms.recordings_updater import (
     RecordingsDataTypeEnum,
 )
 from frigate.comms.review_updater import ReviewDataSubscriber
-from frigate.config import FrigateConfig
+from frigate.config import CameraConfig, FrigateConfig
 from frigate.config.camera.camera import CameraTypeEnum
 from frigate.config.camera.updater import (
     CameraConfigUpdateEnum,
@@ -329,7 +329,10 @@ class EmbeddingMaintainer(threading.Thread):
         camera_config = self.config.cameras[camera]
 
         # no need to process updated objects if face recognition, lpr, genai are disabled
-        if not camera_config.genai.enabled and len(self.realtime_processors) == 0:
+        if (
+            not camera_config.objects.genai.enabled
+            and len(self.realtime_processors) == 0
+        ):
             return
 
         # Create our own thumbnail based on the bounding box and the frame time
@@ -367,23 +370,23 @@ class EmbeddingMaintainer(threading.Thread):
         # check if we're configured to send an early request after a minimum number of updates received
         if (
             self.genai_client is not None
-            and camera_config.genai.send_triggers.after_significant_updates
+            and camera_config.objects.genai.send_triggers.after_significant_updates
         ):
             if (
                 len(self.tracked_events.get(data["id"], []))
-                >= camera_config.genai.send_triggers.after_significant_updates
+                >= camera_config.objects.genai.send_triggers.after_significant_updates
                 and data["id"] not in self.early_request_sent
             ):
                 if data["has_clip"] and data["has_snapshot"]:
                     event: Event = Event.get(Event.id == data["id"])
 
                     if (
-                        not camera_config.genai.objects
+                        not camera_config.objects.genai.objects
-                        or event.label in camera_config.genai.objects
+                        or event.label in camera_config.objects.genai.objects
                     ) and (
-                        not camera_config.genai.required_zones
+                        not camera_config.objects.genai.required_zones
                         or set(data["entered_zones"])
-                        & set(camera_config.genai.required_zones)
+                        & set(camera_config.objects.genai.required_zones)
                     ):
                         logger.debug(f"{camera} sending early request to GenAI")
 
@@ -436,16 +439,17 @@ class EmbeddingMaintainer(threading.Thread):
 
                 # Run GenAI
                 if (
-                    camera_config.genai.enabled
+                    camera_config.objects.genai.enabled
-                    and camera_config.genai.send_triggers.tracked_object_end
+                    and camera_config.objects.genai.send_triggers.tracked_object_end
                     and self.genai_client is not None
                     and (
-                        not camera_config.genai.objects
+                        not camera_config.objects.genai.objects
-                        or event.label in camera_config.genai.objects
+                        or event.label in camera_config.objects.genai.objects
                     )
                     and (
-                        not camera_config.genai.required_zones
+                        not camera_config.objects.genai.required_zones
-                        or set(event.zones) & set(camera_config.genai.required_zones)
+                        or set(event.zones)
+                        & set(camera_config.objects.genai.required_zones)
                     )
                 ):
                     self._process_genai_description(event, camera_config, thumbnail)
@@ -624,8 +628,10 @@ class EmbeddingMaintainer(threading.Thread):
 
         self.embeddings.embed_thumbnail(event_id, thumbnail)
 
-    def _process_genai_description(self, event, camera_config, thumbnail) -> None:
+    def _process_genai_description(
-        if event.has_snapshot and camera_config.genai.use_snapshot:
+        self, event: Event, camera_config: CameraConfig, thumbnail
+    ) -> None:
+        if event.has_snapshot and camera_config.objects.genai.use_snapshot:
             snapshot_image = self._read_and_crop_snapshot(event, camera_config)
             if not snapshot_image:
                 return
@@ -637,7 +643,7 @@ class EmbeddingMaintainer(threading.Thread):
 
         embed_image = (
             [snapshot_image]
-            if event.has_snapshot and camera_config.genai.use_snapshot
+            if event.has_snapshot and camera_config.objects.genai.use_snapshot
             else (
                 [data["thumbnail"] for data in self.tracked_events[event.id]]
                 if num_thumbnails > 0
@@ -645,7 +651,7 @@ class EmbeddingMaintainer(threading.Thread):
             )
         )
 
-        if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
+        if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
             logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}")
 
             Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir(
@@ -775,7 +781,7 @@ class EmbeddingMaintainer(threading.Thread):
             return
 
         camera_config = self.config.cameras[event.camera]
-        if not camera_config.genai.enabled and not force:
+        if not camera_config.objects.genai.enabled and not force:
             logger.error(f"GenAI not enabled for camera {event.camera}")
             return
 

frigate/genai/__init__.py

@@ -40,9 +40,9 @@ class GenAIClient:
         event: Event,
     ) -> Optional[str]:
         """Generate a description for the frame."""
-        prompt = camera_config.genai.object_prompts.get(
+        prompt = camera_config.objects.genai.object_prompts.get(
             event.label,
-            camera_config.genai.prompt,
+            camera_config.objects.genai.prompt,
         ).format(**model_to_dict(event))
         logger.debug(f"Sending images to genai provider with prompt: {prompt}")
         return self._send(prompt, thumbnails)
@@ -58,16 +58,10 @@ class GenAIClient:
 
 def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
     """Get the GenAI client."""
-    genai_config = config.genai
-    genai_cameras = [
-        c for c in config.cameras.values() if c.enabled and c.genai.enabled
-    ]
-
-    if genai_cameras or genai_config.enabled:
     load_providers()
-        provider = PROVIDERS.get(genai_config.provider)
+    provider = PROVIDERS.get(config.genai.provider)
     if provider:
-            return provider(genai_config)
+        return provider(config.genai)
 
     return None
 

frigate/models.py

@@ -13,7 +13,7 @@ from peewee import (
 from playhouse.sqlite_ext import JSONField
 
 
-class Event(Model):  # type: ignore[misc]
+class Event(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     label = CharField(index=True, max_length=20)
     sub_label = CharField(max_length=100, null=True)
@@ -51,7 +51,7 @@ class Event(Model):  # type: ignore[misc]
     data = JSONField()  # ex: tracked object box, region, etc.
 
 
-class Timeline(Model):  # type: ignore[misc]
+class Timeline(Model):
     timestamp = DateTimeField()
     camera = CharField(index=True, max_length=20)
     source = CharField(index=True, max_length=20)  # ex: tracked object, audio, external
@@ -60,13 +60,13 @@ class Timeline(Model):  # type: ignore[misc]
     data = JSONField()  # ex: tracked object id, region, box, etc.
 
 
-class Regions(Model):  # type: ignore[misc]
+class Regions(Model):
     camera = CharField(null=False, primary_key=True, max_length=20)
     grid = JSONField()  # json blob of grid
     last_update = DateTimeField()
 
 
-class Recordings(Model):  # type: ignore[misc]
+class Recordings(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     path = CharField(unique=True)
@@ -80,7 +80,7 @@ class Recordings(Model):  # type: ignore[misc]
     regions = IntegerField(null=True)
 
 
-class Export(Model):  # type: ignore[misc]
+class Export(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     name = CharField(index=True, max_length=100)
@@ -90,7 +90,7 @@ class Export(Model):  # type: ignore[misc]
     in_progress = BooleanField()
 
 
-class ReviewSegment(Model):  # type: ignore[misc]
+class ReviewSegment(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     start_time = DateTimeField()
@@ -100,7 +100,7 @@ class ReviewSegment(Model):  # type: ignore[misc]
     data = JSONField()  # additional data about detection like list of labels, zone, areas of significant motion
 
 
-class UserReviewStatus(Model):  # type: ignore[misc]
+class UserReviewStatus(Model):
     user_id = CharField(max_length=30)
     review_segment = ForeignKeyField(ReviewSegment, backref="user_reviews")
     has_been_reviewed = BooleanField(default=False)
@@ -109,7 +109,7 @@ class UserReviewStatus(Model):  # type: ignore[misc]
         indexes = ((("user_id", "review_segment"), True),)
 
 
-class Previews(Model):  # type: ignore[misc]
+class Previews(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     path = CharField(unique=True)
@@ -119,14 +119,14 @@ class Previews(Model):  # type: ignore[misc]
 
 
 # Used for temporary table in record/cleanup.py
-class RecordingsToDelete(Model):  # type: ignore[misc]
+class RecordingsToDelete(Model):
     id = CharField(null=False, primary_key=False, max_length=30)
 
     class Meta:
         temporary = True
 
 
-class User(Model):  # type: ignore[misc]
+class User(Model):
     username = CharField(null=False, primary_key=True, max_length=30)
     role = CharField(
         max_length=20,
@@ -136,7 +136,7 @@ class User(Model):  # type: ignore[misc]
     notification_tokens = JSONField()
 
 
-class Trigger(Model):  # type: ignore[misc]
+class Trigger(Model):
     camera = CharField(max_length=20)
     name = CharField()
     type = CharField(max_length=10)

frigate/util/config.py

@@ -371,6 +371,22 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
         del new_config["record"]["retain"]
 
+    # migrate global genai to new objects config
+    global_genai = config.get("genai", {})
+
+    if global_genai:
+        new_genai_config = {}
+        new_object_config = config.get("objects", {})
+        new_object_config["genai"] = {}
+
+        for key in global_genai.keys():
+            if key not in ["provider", "base_url", "api_key"]:
+                new_object_config["genai"][key] = global_genai[key]
+            else:
+                new_genai_config[key] = global_genai[key]
+
+        config["genai"] = new_genai_config
+
     for name, camera in config.get("cameras", {}).items():
         camera_config: dict[str, dict[str, Any]] = camera.copy()
         camera_record_retain = camera_config.get("record", {}).get("retain")
@@ -392,6 +408,13 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
             del camera_config["record"]["retain"]
 
+        camera_genai = camera_config.get("genai", {})
+
+        if camera_genai:
+            new_object_config = config.get("objects", {})
+            new_object_config["genai"] = camera_genai
+            del camera_config["genai"]
+
         new_config["cameras"][name] = camera_config
 
     new_config["version"] = "0.17-0"

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -936,14 +936,17 @@ function ObjectDetailsTab({
         </div>
       </div>
       <div className="flex flex-col gap-1.5">
-        {config?.cameras[search.camera].genai.enabled &&
+        {config?.cameras[search.camera].objects.genai.enabled &&
         !search.end_time &&
-        (config.cameras[search.camera].genai.required_zones.length === 0 ||
+        (config.cameras[search.camera].objects.genai.required_zones.length ===
+          0 ||
           search.zones.some((zone) =>
-            config.cameras[search.camera].genai.required_zones.includes(zone),
+            config.cameras[search.camera].objects.genai.required_zones.includes(
+              zone,
+            ),
           )) &&
-        (config.cameras[search.camera].genai.objects.length === 0 ||
+        (config.cameras[search.camera].objects.genai.objects.length === 0 ||
-          config.cameras[search.camera].genai.objects.includes(
+          config.cameras[search.camera].objects.genai.objects.includes(
             search.label,
           )) ? (
           <>
@@ -972,7 +975,8 @@ function ObjectDetailsTab({
         )}
 
         <div className="flex w-full flex-row justify-end gap-2">
-          {config?.cameras[search.camera].genai.enabled && search.end_time && (
+          {config?.cameras[search.camera].objects.genai.enabled &&
+            search.end_time && (
               <div className="flex items-start">
                 <Button
                   className="rounded-r-none border-r-0"
@@ -1011,8 +1015,9 @@ function ObjectDetailsTab({
                 )}
               </div>
             )}
-          {((config?.cameras[search.camera].genai.enabled && search.end_time) ||
+          {((config?.cameras[search.camera].objects.genai.enabled &&
-            !config?.cameras[search.camera].genai.enabled) && (
+            search.end_time) ||
+            !config?.cameras[search.camera].objects.genai.enabled) && (
             <Button
               variant="select"
               aria-label={t("button.save", { ns: "common" })}

web/src/types/frigateConfig.ts

@@ -94,13 +94,6 @@ export interface CameraConfig {
     cmd: string;
     roles: string[];
   }[];
-  genai: {
-    enabled: string;
-    prompt: string;
-    object_prompts: { [key: string]: string };
-    required_zones: string[];
-    objects: string[];
-  };
   live: {
     height: number;
     quality: number;
@@ -146,6 +139,14 @@ export interface CameraConfig {
     };
     mask: string;
     track: string[];
+    genai: {
+      enabled: boolean;
+      enabled_in_config: boolean;
+      prompt: string;
+      object_prompts: { [key: string]: string };
+      required_zones: string[];
+      objects: string[];
+    };
   };
   onvif: {
     autotracking: {
@@ -406,15 +407,10 @@ export interface FrigateConfig {
   };
 
   genai: {
-    enabled: boolean;
     provider: string;
     base_url?: string;
     api_key?: string;
     model: string;
-    prompt: string;
-    object_prompts: { [key: string]: string };
-    required_zones: string[];
-    objects: string[];
   };
 
   go2rtc: {

web/src/views/settings/CameraSettingsView.tsx

@@ -413,7 +413,7 @@ export default function CameraSettingsView({
                   </div>
                 </div>
               </div>
-              {config?.genai?.enabled && (
+              {cameraConfig?.objects?.genai?.enabled_in_config && (
                 <>
                   <Separator className="my-2 flex bg-secondary" />