Migrate object genai configuration (#19437)

* Move genAI object to objects section

* Adjust config propogation behavior

* Refactor genai config usage

* Automatic migration

* Always start the embeddings process

* Always init embeddings

* Config fixes

* Adjust reference config

* Adjust docs

* Formatting

* Fix

.github/workflows/pull_request.yml

@@ -107,7 +107,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Build
-        run: make
+        run: make debug
       - name: Run mypy
         run: docker run --rm --entrypoint=python3 frigate:latest -u -m mypy --config-file frigate/mypy.ini frigate
       - name: Run tests

Makefile

@@ -20,6 +20,12 @@ local: version
 		--tag frigate:latest \
 		--load
 
+debug: version
+	docker buildx build --target=frigate --file docker/main/Dockerfile . \
+	    --build-arg DEBUG=true \
+		--tag frigate:latest \
+		--load
+
 amd64:
 	docker buildx build --target=frigate --file docker/main/Dockerfile . \
 		--tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH) \

docker/main/Dockerfile

@@ -148,6 +148,7 @@ RUN --mount=type=bind,source=docker/main/install_s6_overlay.sh,target=/deps/inst
 FROM base AS wheels
 ARG DEBIAN_FRONTEND
 ARG TARGETARCH
+ARG DEBUG=false
 
 # Use a separate container to build wheels to prevent build dependencies in final image
 RUN apt-get -qq update \
@@ -177,6 +178,8 @@ RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \
     && python3 get-pip.py "pip"
 
 COPY docker/main/requirements.txt /requirements.txt
+COPY docker/main/requirements-dev.txt /requirements-dev.txt
+
 RUN pip3 install -r /requirements.txt
 
 # Build pysqlite3 from source
@@ -184,7 +187,10 @@ COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh
 RUN /build_pysqlite3.sh
 
 COPY docker/main/requirements-wheels.txt /requirements-wheels.txt
-RUN pip3 wheel --wheel-dir=/wheels -r /requirements-wheels.txt
+RUN pip3 wheel --wheel-dir=/wheels -r /requirements-wheels.txt && \
+    if [ "$DEBUG" = "true" ]; then \
+        pip3 wheel --wheel-dir=/wheels -r /requirements-dev.txt; \
+    fi
 
 # Install HailoRT & Wheels
 RUN --mount=type=bind,source=docker/main/install_hailort.sh,target=/deps/install_hailort.sh \

docker/main/requirements-dev.txt

@@ -1 +1,4 @@
 ruff
+
+# types
+types-peewee == 3.17.*

docs/docs/configuration/genai.md

@@ -15,23 +15,24 @@ To use Generative AI, you must define a single provider at the global level of y
 
 ```yaml
 genai:
-  enabled: True
   provider: gemini
   api_key: "{FRIGATE_GEMINI_API_KEY}"
   model: gemini-1.5-flash
 
 cameras:
   front_camera:
+    objects:
     genai:
-      enabled: True # <- enable GenAI for your front camera
+        enabled: True # <- enable GenAI for your front camera
-      use_snapshot: True
+        use_snapshot: True
-      objects:
+        objects:
-        - person
+          - person
-      required_zones:
+        required_zones:
-        - steps
+          - steps
   indoor_camera:
-    genai:
+    objects:
-      enabled: False # <- disable GenAI for your indoor camera
+      genai:
+        enabled: False # <- disable GenAI for your indoor camera
 ```
 
 By default, descriptions will be generated for all tracked objects and all zones. But you can also optionally specify `objects` and `required_zones` to only generate descriptions for certain tracked objects or zones.
@@ -68,7 +69,6 @@ You should have at least 8 GB of RAM available (or VRAM if running on GPU) to ru
 
 ```yaml
 genai:
-  enabled: True
   provider: ollama
   base_url: http://localhost:11434
   model: llava:7b
@@ -95,7 +95,6 @@ To start using Gemini, you must first get an API key from [Google AI Studio](htt
 
 ```yaml
 genai:
-  enabled: True
   provider: gemini
   api_key: "{FRIGATE_GEMINI_API_KEY}"
   model: gemini-1.5-flash
@@ -117,7 +116,6 @@ To start using OpenAI, you must first [create an API key](https://platform.opena
 
 ```yaml
 genai:
-  enabled: True
   provider: openai
   api_key: "{FRIGATE_OPENAI_API_KEY}"
   model: gpt-4o
@@ -145,7 +143,6 @@ To start using Azure OpenAI, you must first [create a resource](https://learn.mi
 
 ```yaml
 genai:
-  enabled: True
   provider: azure_openai
   base_url: https://example-endpoint.openai.azure.com/openai/deployments/gpt-4o/chat/completions?api-version=2023-03-15-preview
   api_key: "{FRIGATE_OPENAI_API_KEY}"
@@ -188,32 +185,35 @@ You are also able to define custom prompts in your configuration.
 
 ```yaml
 genai:
-  enabled: True
   provider: ollama
   base_url: http://localhost:11434
   model: llava
-  prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
+
-  object_prompts:
+objects:
-    person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
+    prompt: "Analyze the {label} in these images from the {camera} security camera. Focus on the actions, behavior, and potential intent of the {label}, rather than just describing its appearance."
-    car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
+    object_prompts:
+      person: "Examine the main person in these images. What are they doing and what might their actions suggest about their intent (e.g., approaching a door, leaving an area, standing still)? Do not describe the surroundings or static details."
+      car: "Observe the primary vehicle in these images. Focus on its movement, direction, or purpose (e.g., parking, approaching, circling). If it's a delivery vehicle, mention the company."
 ```
 
-Prompts can also be overriden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
+Prompts can also be overridden at the camera level to provide a more detailed prompt to the model about your specific camera, if you desire.
 
 ```yaml
 cameras:
   front_door:
-    genai:
+    objects:
-      use_snapshot: True
+      genai:
-      prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
+        enabled: True
-      object_prompts:
+        use_snapshot: True
-        person: "Examine the person in these images. What are they doing, and how might their actions suggest their purpose (e.g., delivering something, approaching, leaving)? If they are carrying or interacting with a package, include details about its source or destination."
+        prompt: "Analyze the {label} in these images from the {camera} security camera at the front door. Focus on the actions and potential intent of the {label}."
-        cat: "Observe the cat in these images. Focus on its movement and intent (e.g., wandering, hunting, interacting with objects). If the cat is near the flower pots or engaging in any specific actions, mention it."
+        object_prompts:
-      objects:
+          person: "Examine the person in these images. What are they doing, and how might their actions suggest their purpose (e.g., delivering something, approaching, leaving)? If they are carrying or interacting with a package, include details about its source or destination."
-        - person
+          cat: "Observe the cat in these images. Focus on its movement and intent (e.g., wandering, hunting, interacting with objects). If the cat is near the flower pots or engaging in any specific actions, mention it."
-        - cat
+        objects:
-      required_zones:
+          - person
-        - steps
+          - cat
+        required_zones:
+          - steps
 ```
 
 ### Experiment with prompts

docs/docs/configuration/reference.md

@@ -339,6 +339,33 @@ objects:
       # Optional: mask to prevent this object type from being detected in certain areas (default: no mask)
       # Checks based on the bottom center of the bounding box of the object
       mask: 0.000,0.000,0.781,0.000,0.781,0.278,0.000,0.278
+  # Optional: Configuration for AI generated tracked object descriptions
+  genai:
+    # Optional: Enable AI object description generation (default: shown below)
+    enabled: False
+    # Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
+    use_snapshot: False
+    # Optional: The default prompt for generating descriptions. Can use replacement
+    # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
+    prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
+    # Optional: Object specific prompts to customize description results
+    # Format: {label}: {prompt}
+    object_prompts:
+      person: "My special person prompt."
+    # Optional: objects to generate descriptions for (default: all objects that are tracked)
+    objects:
+      - person
+      - cat
+    # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
+    required_zones: []
+    # Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
+    send_triggers:
+      # Once the object is no longer tracked
+      tracked_object_end: True
+      # Optional: After X many significant updates are received (default: shown below)
+      after_significant_updates: None
+    # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
+    debug_save_thumbnails: False
 
 # Optional: Review configuration
 # NOTE: Can be overridden at the camera level
@@ -612,13 +639,6 @@ genai:
   base_url: http://localhost::11434
   # Required if gemini or openai
   api_key: "{FRIGATE_GENAI_API_KEY}"
-  # Optional: The default prompt for generating descriptions. Can use replacement
-  # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
-  prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
-  # Optional: Object specific prompts to customize description results
-  # Format: {label}: {prompt}
-  object_prompts:
-    person: "My special person prompt."
 
 # Optional: Configuration for audio transcription
 # NOTE: only the enabled option can be overridden at the camera level
@@ -857,34 +877,6 @@ cameras:
         actions:
           - notification
 
-    # Optional: Configuration for AI generated tracked object descriptions
-    genai:
-      # Optional: Enable AI description generation (default: shown below)
-      enabled: False
-      # Optional: Use the object snapshot instead of thumbnails for description generation (default: shown below)
-      use_snapshot: False
-      # Optional: The default prompt for generating descriptions. Can use replacement
-      # variables like "label", "sub_label", "camera" to make more dynamic. (default: shown below)
-      prompt: "Describe the {label} in the sequence of images with as much detail as possible. Do not describe the background."
-      # Optional: Object specific prompts to customize description results
-      # Format: {label}: {prompt}
-      object_prompts:
-        person: "My special person prompt."
-      # Optional: objects to generate descriptions for (default: all objects that are tracked)
-      objects:
-        - person
-        - cat
-      # Optional: Restrict generation to objects that entered any of the listed zones (default: none, all zones qualify)
-      required_zones: []
-      # Optional: What triggers to use to send frames for a tracked object to generative AI (default: shown below)
-      send_triggers:
-        # Once the object is no longer tracked
-        tracked_object_end: True
-        # Optional: After X many significant updates are received (default: shown below)
-        after_significant_updates: None
-      # Optional: Save thumbnails sent to generative AI for review/debugging purposes (default: shown below)
-      debug_save_thumbnails: False
-
 # Optional
 ui:
   # Optional: Set a timezone to use in the UI (default: use browser local time)

frigate/api/event.py

@@ -1230,7 +1230,7 @@ def regenerate_description(
 
     camera_config = request.app.frigate_config.cameras[event.camera]
 
-    if camera_config.genai.enabled or params.force:
+    if camera_config.objects.genai.enabled or params.force:
         request.app.event_metadata_updater.publish(
             (event.id, params.source, params.force),
             EventMetadataTypeEnum.regenerate_description.value,

frigate/app.py

@@ -246,18 +246,7 @@ class FrigateApp:
         logger.info(f"Review process started: {review_segment_process.pid}")
 
     def init_embeddings_manager(self) -> None:
-        genai_cameras = [
+        # always start the embeddings process
-            c for c in self.config.cameras.values() if c.enabled and c.genai.enabled
-        ]
-
-        if (
-            not self.config.semantic_search.enabled
-            and not genai_cameras
-            and not self.config.lpr.enabled
-            and not self.config.face_recognition.enabled
-        ):
-            return
-
         embedding_process = EmbeddingProcess(
             self.config, self.embeddings_metrics, self.stop_event
         )
@@ -309,20 +298,8 @@ class FrigateApp:
             migrate_exports(self.config.ffmpeg, list(self.config.cameras.keys()))
 
     def init_embeddings_client(self) -> None:
-        genai_cameras = [
+        # Create a client for other processes to use
-            c
+        self.embeddings = EmbeddingsContext(self.db)
-            for c in self.config.cameras.values()
-            if c.enabled_in_config and c.genai.enabled
-        ]
-
-        if (
-            self.config.semantic_search.enabled
-            or self.config.lpr.enabled
-            or genai_cameras
-            or self.config.face_recognition.enabled
-        ):
-            # Create a client for other processes to use
-            self.embeddings = EmbeddingsContext(self.db)
 
     def init_inter_process_communicator(self) -> None:
         self.inter_process_communicator = InterProcessCommunicator()

frigate/comms/dispatcher.py

@@ -3,7 +3,7 @@
 import datetime
 import json
 import logging
-from typing import Any, Callable, Optional
+from typing import Any, Callable, Optional, cast
 
 from frigate.camera import PTZMetrics
 from frigate.camera.activity_manager import CameraActivityManager
@@ -135,7 +135,7 @@ class Dispatcher:
 
         def handle_update_event_description() -> None:
             event: Event = Event.get(Event.id == payload["id"])
-            event.data["description"] = payload["description"]
+            cast(dict, event.data)["description"] = payload["description"]
             event.save()
             self.publish(
                 "tracked_object_update",
@@ -209,7 +209,7 @@ class Dispatcher:
                     ].onvif.autotracking.enabled,
                     "alerts": self.config.cameras[camera].review.alerts.enabled,
                     "detections": self.config.cameras[camera].review.detections.enabled,
-                    "genai": self.config.cameras[camera].genai.enabled,
+                    "genai": self.config.cameras[camera].objects.genai.enabled,
                 }
 
             self.publish("camera_activity", json.dumps(camera_status))
@@ -744,10 +744,10 @@ class Dispatcher:
 
     def _on_genai_command(self, camera_name: str, payload: str) -> None:
         """Callback for GenAI topic."""
-        genai_settings = self.config.cameras[camera_name].genai
+        genai_settings = self.config.cameras[camera_name].objects.genai
 
         if payload == "ON":
-            if not self.config.cameras[camera_name].genai.enabled_in_config:
+            if not self.config.cameras[camera_name].objects.genai.enabled_in_config:
                 logger.error(
                     "GenAI must be enabled in the config to be turned on via MQTT."
                 )

frigate/comms/mqtt.py

@@ -124,7 +124,7 @@ class MqttClient(Communicator):
             )
             self.publish(
                 f"{camera_name}/genai/state",
-                "ON" if camera.genai.enabled_in_config else "OFF",
+                "ON" if camera.objects.genai.enabled_in_config else "OFF",
                 retain=True,
             )
 

frigate/comms/webpush.py

@@ -70,7 +70,7 @@ class WebPushClient(Communicator):
         # Pull keys from PEM or generate if they do not exist
         self.vapid = Vapid01.from_file(os.path.join(CONFIG_DIR, "notifications.pem"))
 
-        users: list[User] = (
+        users: list[dict[str, Any]] = (
             User.select(User.username, User.notification_tokens).dicts().iterator()
         )
         for user in users:

frigate/config/camera/camera.py

@@ -28,7 +28,6 @@ from .audio import AudioConfig
 from .birdseye import BirdseyeCameraConfig
 from .detect import DetectConfig
 from .ffmpeg import CameraFfmpegConfig, CameraInput
-from .genai import GenAICameraConfig
 from .live import CameraLiveConfig
 from .motion import MotionConfig
 from .mqtt import CameraMqttConfig
@@ -71,9 +70,6 @@ class CameraConfig(FrigateBaseModel):
         default_factory=CameraFaceRecognitionConfig, title="Face recognition config."
     )
     ffmpeg: CameraFfmpegConfig = Field(title="FFmpeg configuration for the camera.")
-    genai: GenAICameraConfig = Field(
-        default_factory=GenAICameraConfig, title="Generative AI configuration."
-    )
     live: CameraLiveConfig = Field(
         default_factory=CameraLiveConfig, title="Live playback settings."
     )

frigate/config/camera/genai.py

@@ -1,12 +1,12 @@
 from enum import Enum
-from typing import Optional, Union
+from typing import Optional
 
-from pydantic import BaseModel, Field, field_validator
+from pydantic import Field
 
 from ..base import FrigateBaseModel
 from ..env import EnvString
 
-__all__ = ["GenAIConfig", "GenAICameraConfig", "GenAIProviderEnum"]
+__all__ = ["GenAIConfig", "GenAIProviderEnum"]
 
 
 class GenAIProviderEnum(str, Enum):
@@ -16,70 +16,8 @@ class GenAIProviderEnum(str, Enum):
     ollama = "ollama"
 
 
-class GenAISendTriggersConfig(BaseModel):
-    tracked_object_end: bool = Field(
-        default=True, title="Send once the object is no longer tracked."
-    )
-    after_significant_updates: Optional[int] = Field(
-        default=None,
-        title="Send an early request to generative AI when X frames accumulated.",
-        ge=1,
-    )
-
-
-# uses BaseModel because some global attributes are not available at the camera level
-class GenAICameraConfig(BaseModel):
-    enabled: bool = Field(default=False, title="Enable GenAI for camera.")
-    use_snapshot: bool = Field(
-        default=False, title="Use snapshots for generating descriptions."
-    )
-    prompt: str = Field(
-        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
-        title="Default caption prompt.",
-    )
-    object_prompts: dict[str, str] = Field(
-        default_factory=dict, title="Object specific prompts."
-    )
-
-    objects: Union[str, list[str]] = Field(
-        default_factory=list,
-        title="List of objects to run generative AI for.",
-    )
-    required_zones: Union[str, list[str]] = Field(
-        default_factory=list,
-        title="List of required zones to be entered in order to run generative AI.",
-    )
-    debug_save_thumbnails: bool = Field(
-        default=False,
-        title="Save thumbnails sent to generative AI for debugging purposes.",
-    )
-    send_triggers: GenAISendTriggersConfig = Field(
-        default_factory=GenAISendTriggersConfig,
-        title="What triggers to use to send frames to generative AI for a tracked object.",
-    )
-
-    enabled_in_config: Optional[bool] = Field(
-        default=None, title="Keep track of original state of generative AI."
-    )
-
-    @field_validator("required_zones", mode="before")
-    @classmethod
-    def validate_required_zones(cls, v):
-        if isinstance(v, str) and "," not in v:
-            return [v]
-
-        return v
-
-
 class GenAIConfig(FrigateBaseModel):
-    enabled: bool = Field(default=False, title="Enable GenAI.")
+    """Primary GenAI Config to define GenAI Provider."""
-    prompt: str = Field(
-        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
-        title="Default caption prompt.",
-    )
-    object_prompts: dict[str, str] = Field(
-        default_factory=dict, title="Object specific prompts."
-    )
 
     api_key: Optional[EnvString] = Field(default=None, title="Provider API key.")
     base_url: Optional[str] = Field(default=None, title="Provider base url.")

frigate/config/camera/objects.py

@@ -1,10 +1,10 @@
 from typing import Any, Optional, Union
 
-from pydantic import Field, PrivateAttr, field_serializer
+from pydantic import Field, PrivateAttr, field_serializer, field_validator
 
 from ..base import FrigateBaseModel
 
-__all__ = ["ObjectConfig", "FilterConfig"]
+__all__ = ["ObjectConfig", "GenAIObjectConfig", "FilterConfig"]
 
 
 DEFAULT_TRACKED_OBJECTS = ["person"]
@@ -49,12 +49,69 @@ class FilterConfig(FrigateBaseModel):
         return None
 
 
+class GenAIObjectTriggerConfig(FrigateBaseModel):
+    tracked_object_end: bool = Field(
+        default=True, title="Send once the object is no longer tracked."
+    )
+    after_significant_updates: Optional[int] = Field(
+        default=None,
+        title="Send an early request to generative AI when X frames accumulated.",
+        ge=1,
+    )
+
+
+class GenAIObjectConfig(FrigateBaseModel):
+    enabled: bool = Field(default=False, title="Enable GenAI for camera.")
+    use_snapshot: bool = Field(
+        default=False, title="Use snapshots for generating descriptions."
+    )
+    prompt: str = Field(
+        default="Analyze the sequence of images containing the {label}. Focus on the likely intent or behavior of the {label} based on its actions and movement, rather than describing its appearance or the surroundings. Consider what the {label} is doing, why, and what it might do next.",
+        title="Default caption prompt.",
+    )
+    object_prompts: dict[str, str] = Field(
+        default_factory=dict, title="Object specific prompts."
+    )
+
+    objects: Union[str, list[str]] = Field(
+        default_factory=list,
+        title="List of objects to run generative AI for.",
+    )
+    required_zones: Union[str, list[str]] = Field(
+        default_factory=list,
+        title="List of required zones to be entered in order to run generative AI.",
+    )
+    debug_save_thumbnails: bool = Field(
+        default=False,
+        title="Save thumbnails sent to generative AI for debugging purposes.",
+    )
+    send_triggers: GenAIObjectTriggerConfig = Field(
+        default_factory=GenAIObjectTriggerConfig,
+        title="What triggers to use to send frames to generative AI for a tracked object.",
+    )
+    enabled_in_config: Optional[bool] = Field(
+        default=None, title="Keep track of original state of generative AI."
+    )
+
+    @field_validator("required_zones", mode="before")
+    @classmethod
+    def validate_required_zones(cls, v):
+        if isinstance(v, str) and "," not in v:
+            return [v]
+
+        return v
+
+
 class ObjectConfig(FrigateBaseModel):
     track: list[str] = Field(default=DEFAULT_TRACKED_OBJECTS, title="Objects to track.")
     filters: dict[str, FilterConfig] = Field(
         default_factory=dict, title="Object filters."
     )
     mask: Union[str, list[str]] = Field(default="", title="Object mask.")
+    genai: GenAIObjectConfig = Field(
+        default_factory=GenAIObjectConfig,
+        title="Config for using genai to analyze objects.",
+    )
     _all_objects: list[str] = PrivateAttr()
 
     @property

frigate/config/camera/updater.py

@@ -99,7 +99,7 @@ class CameraConfigUpdateSubscriber:
         elif update_type == CameraConfigUpdateEnum.enabled:
             config.enabled = updated_config
         elif update_type == CameraConfigUpdateEnum.genai:
-            config.genai = updated_config
+            config.objects.genai = updated_config
         elif update_type == CameraConfigUpdateEnum.motion:
             config.motion = updated_config
         elif update_type == CameraConfigUpdateEnum.notifications:

frigate/config/config.py

@@ -352,6 +352,11 @@ class FrigateConfig(FrigateBaseModel):
         default_factory=ModelConfig, title="Detection model configuration."
     )
 
+    # GenAI config
+    genai: GenAIConfig = Field(
+        default_factory=GenAIConfig, title="Generative AI configuration."
+    )
+
     # Camera config
     cameras: Dict[str, CameraConfig] = Field(title="Camera configuration.")
     audio: AudioConfig = Field(
@@ -366,9 +371,6 @@ class FrigateConfig(FrigateBaseModel):
     ffmpeg: FfmpegConfig = Field(
         default_factory=FfmpegConfig, title="Global FFmpeg configuration."
     )
-    genai: GenAIConfig = Field(
-        default_factory=GenAIConfig, title="Generative AI configuration."
-    )
     live: CameraLiveConfig = Field(
         default_factory=CameraLiveConfig, title="Live playback settings."
     )
@@ -458,7 +460,6 @@ class FrigateConfig(FrigateBaseModel):
                 "live": ...,
                 "objects": ...,
                 "review": ...,
-                "genai": ...,
                 "motion": ...,
                 "notifications": ...,
                 "detect": ...,
@@ -606,7 +607,9 @@ class FrigateConfig(FrigateBaseModel):
             camera_config.review.detections.enabled_in_config = (
                 camera_config.review.detections.enabled
             )
-            camera_config.genai.enabled_in_config = camera_config.genai.enabled
+            camera_config.objects.genai.enabled_in_config = (
+                camera_config.objects.genai.enabled
+            )
 
             # Add default filters
             object_keys = camera_config.objects.track

frigate/embeddings/maintainer.py

@@ -30,7 +30,7 @@ from frigate.comms.recordings_updater import (
     RecordingsDataTypeEnum,
 )
 from frigate.comms.review_updater import ReviewDataSubscriber
-from frigate.config import FrigateConfig
+from frigate.config import CameraConfig, FrigateConfig
 from frigate.config.camera.camera import CameraTypeEnum
 from frigate.config.camera.updater import (
     CameraConfigUpdateEnum,
@@ -329,7 +329,10 @@ class EmbeddingMaintainer(threading.Thread):
         camera_config = self.config.cameras[camera]
 
         # no need to process updated objects if face recognition, lpr, genai are disabled
-        if not camera_config.genai.enabled and len(self.realtime_processors) == 0:
+        if (
+            not camera_config.objects.genai.enabled
+            and len(self.realtime_processors) == 0
+        ):
             return
 
         # Create our own thumbnail based on the bounding box and the frame time
@@ -367,23 +370,23 @@ class EmbeddingMaintainer(threading.Thread):
         # check if we're configured to send an early request after a minimum number of updates received
         if (
             self.genai_client is not None
-            and camera_config.genai.send_triggers.after_significant_updates
+            and camera_config.objects.genai.send_triggers.after_significant_updates
         ):
             if (
                 len(self.tracked_events.get(data["id"], []))
-                >= camera_config.genai.send_triggers.after_significant_updates
+                >= camera_config.objects.genai.send_triggers.after_significant_updates
                 and data["id"] not in self.early_request_sent
             ):
                 if data["has_clip"] and data["has_snapshot"]:
                     event: Event = Event.get(Event.id == data["id"])
 
                     if (
-                        not camera_config.genai.objects
+                        not camera_config.objects.genai.objects
-                        or event.label in camera_config.genai.objects
+                        or event.label in camera_config.objects.genai.objects
                     ) and (
-                        not camera_config.genai.required_zones
+                        not camera_config.objects.genai.required_zones
                         or set(data["entered_zones"])
-                        & set(camera_config.genai.required_zones)
+                        & set(camera_config.objects.genai.required_zones)
                     ):
                         logger.debug(f"{camera} sending early request to GenAI")
 
@@ -436,16 +439,17 @@ class EmbeddingMaintainer(threading.Thread):
 
                 # Run GenAI
                 if (
-                    camera_config.genai.enabled
+                    camera_config.objects.genai.enabled
-                    and camera_config.genai.send_triggers.tracked_object_end
+                    and camera_config.objects.genai.send_triggers.tracked_object_end
                     and self.genai_client is not None
                     and (
-                        not camera_config.genai.objects
+                        not camera_config.objects.genai.objects
-                        or event.label in camera_config.genai.objects
+                        or event.label in camera_config.objects.genai.objects
                     )
                     and (
-                        not camera_config.genai.required_zones
+                        not camera_config.objects.genai.required_zones
-                        or set(event.zones) & set(camera_config.genai.required_zones)
+                        or set(event.zones)
+                        & set(camera_config.objects.genai.required_zones)
                     )
                 ):
                     self._process_genai_description(event, camera_config, thumbnail)
@@ -624,8 +628,10 @@ class EmbeddingMaintainer(threading.Thread):
 
         self.embeddings.embed_thumbnail(event_id, thumbnail)
 
-    def _process_genai_description(self, event, camera_config, thumbnail) -> None:
+    def _process_genai_description(
-        if event.has_snapshot and camera_config.genai.use_snapshot:
+        self, event: Event, camera_config: CameraConfig, thumbnail
+    ) -> None:
+        if event.has_snapshot and camera_config.objects.genai.use_snapshot:
             snapshot_image = self._read_and_crop_snapshot(event, camera_config)
             if not snapshot_image:
                 return
@@ -637,7 +643,7 @@ class EmbeddingMaintainer(threading.Thread):
 
         embed_image = (
             [snapshot_image]
-            if event.has_snapshot and camera_config.genai.use_snapshot
+            if event.has_snapshot and camera_config.objects.genai.use_snapshot
             else (
                 [data["thumbnail"] for data in self.tracked_events[event.id]]
                 if num_thumbnails > 0
@@ -645,7 +651,7 @@ class EmbeddingMaintainer(threading.Thread):
             )
         )
 
-        if camera_config.genai.debug_save_thumbnails and num_thumbnails > 0:
+        if camera_config.objects.genai.debug_save_thumbnails and num_thumbnails > 0:
             logger.debug(f"Saving {num_thumbnails} thumbnails for event {event.id}")
 
             Path(os.path.join(CLIPS_DIR, f"genai-requests/{event.id}")).mkdir(
@@ -775,7 +781,7 @@ class EmbeddingMaintainer(threading.Thread):
             return
 
         camera_config = self.config.cameras[event.camera]
-        if not camera_config.genai.enabled and not force:
+        if not camera_config.objects.genai.enabled and not force:
             logger.error(f"GenAI not enabled for camera {event.camera}")
             return
 

frigate/genai/__init__.py

@@ -40,9 +40,9 @@ class GenAIClient:
         event: Event,
     ) -> Optional[str]:
         """Generate a description for the frame."""
-        prompt = camera_config.genai.object_prompts.get(
+        prompt = camera_config.objects.genai.object_prompts.get(
             event.label,
-            camera_config.genai.prompt,
+            camera_config.objects.genai.prompt,
         ).format(**model_to_dict(event))
         logger.debug(f"Sending images to genai provider with prompt: {prompt}")
         return self._send(prompt, thumbnails)
@@ -58,16 +58,10 @@ class GenAIClient:
 
 def get_genai_client(config: FrigateConfig) -> Optional[GenAIClient]:
     """Get the GenAI client."""
-    genai_config = config.genai
+    load_providers()
-    genai_cameras = [
+    provider = PROVIDERS.get(config.genai.provider)
-        c for c in config.cameras.values() if c.enabled and c.genai.enabled
+    if provider:
-    ]
+        return provider(config.genai)
-
-    if genai_cameras or genai_config.enabled:
-        load_providers()
-        provider = PROVIDERS.get(genai_config.provider)
-        if provider:
-            return provider(genai_config)
 
     return None
 

frigate/models.py

@@ -13,7 +13,7 @@ from peewee import (
 from playhouse.sqlite_ext import JSONField
 
 
-class Event(Model):  # type: ignore[misc]
+class Event(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     label = CharField(index=True, max_length=20)
     sub_label = CharField(max_length=100, null=True)
@@ -51,7 +51,7 @@ class Event(Model):  # type: ignore[misc]
     data = JSONField()  # ex: tracked object box, region, etc.
 
 
-class Timeline(Model):  # type: ignore[misc]
+class Timeline(Model):
     timestamp = DateTimeField()
     camera = CharField(index=True, max_length=20)
     source = CharField(index=True, max_length=20)  # ex: tracked object, audio, external
@@ -60,13 +60,13 @@ class Timeline(Model):  # type: ignore[misc]
     data = JSONField()  # ex: tracked object id, region, box, etc.
 
 
-class Regions(Model):  # type: ignore[misc]
+class Regions(Model):
     camera = CharField(null=False, primary_key=True, max_length=20)
     grid = JSONField()  # json blob of grid
     last_update = DateTimeField()
 
 
-class Recordings(Model):  # type: ignore[misc]
+class Recordings(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     path = CharField(unique=True)
@@ -80,7 +80,7 @@ class Recordings(Model):  # type: ignore[misc]
     regions = IntegerField(null=True)
 
 
-class Export(Model):  # type: ignore[misc]
+class Export(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     name = CharField(index=True, max_length=100)
@@ -90,7 +90,7 @@ class Export(Model):  # type: ignore[misc]
     in_progress = BooleanField()
 
 
-class ReviewSegment(Model):  # type: ignore[misc]
+class ReviewSegment(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     start_time = DateTimeField()
@@ -100,7 +100,7 @@ class ReviewSegment(Model):  # type: ignore[misc]
     data = JSONField()  # additional data about detection like list of labels, zone, areas of significant motion
 
 
-class UserReviewStatus(Model):  # type: ignore[misc]
+class UserReviewStatus(Model):
     user_id = CharField(max_length=30)
     review_segment = ForeignKeyField(ReviewSegment, backref="user_reviews")
     has_been_reviewed = BooleanField(default=False)
@@ -109,7 +109,7 @@ class UserReviewStatus(Model):  # type: ignore[misc]
         indexes = ((("user_id", "review_segment"), True),)
 
 
-class Previews(Model):  # type: ignore[misc]
+class Previews(Model):
     id = CharField(null=False, primary_key=True, max_length=30)
     camera = CharField(index=True, max_length=20)
     path = CharField(unique=True)
@@ -119,14 +119,14 @@ class Previews(Model):  # type: ignore[misc]
 
 
 # Used for temporary table in record/cleanup.py
-class RecordingsToDelete(Model):  # type: ignore[misc]
+class RecordingsToDelete(Model):
     id = CharField(null=False, primary_key=False, max_length=30)
 
     class Meta:
         temporary = True
 
 
-class User(Model):  # type: ignore[misc]
+class User(Model):
     username = CharField(null=False, primary_key=True, max_length=30)
     role = CharField(
         max_length=20,
@@ -136,7 +136,7 @@ class User(Model):  # type: ignore[misc]
     notification_tokens = JSONField()
 
 
-class Trigger(Model):  # type: ignore[misc]
+class Trigger(Model):
     camera = CharField(max_length=20)
     name = CharField()
     type = CharField(max_length=10)

frigate/util/config.py

@@ -371,6 +371,22 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
         del new_config["record"]["retain"]
 
+    # migrate global genai to new objects config
+    global_genai = config.get("genai", {})
+
+    if global_genai:
+        new_genai_config = {}
+        new_object_config = config.get("objects", {})
+        new_object_config["genai"] = {}
+
+        for key in global_genai.keys():
+            if key not in ["provider", "base_url", "api_key"]:
+                new_object_config["genai"][key] = global_genai[key]
+            else:
+                new_genai_config[key] = global_genai[key]
+
+        config["genai"] = new_genai_config
+
     for name, camera in config.get("cameras", {}).items():
         camera_config: dict[str, dict[str, Any]] = camera.copy()
         camera_record_retain = camera_config.get("record", {}).get("retain")
@@ -392,6 +408,13 @@ def migrate_017_0(config: dict[str, dict[str, Any]]) -> dict[str, dict[str, Any]
 
             del camera_config["record"]["retain"]
 
+        camera_genai = camera_config.get("genai", {})
+
+        if camera_genai:
+            new_object_config = config.get("objects", {})
+            new_object_config["genai"] = camera_genai
+            del camera_config["genai"]
+
         new_config["cameras"][name] = camera_config
 
     new_config["version"] = "0.17-0"

web/src/components/overlay/detail/SearchDetailDialog.tsx

@@ -936,14 +936,17 @@ function ObjectDetailsTab({
         </div>
       </div>
       <div className="flex flex-col gap-1.5">
-        {config?.cameras[search.camera].genai.enabled &&
+        {config?.cameras[search.camera].objects.genai.enabled &&
         !search.end_time &&
-        (config.cameras[search.camera].genai.required_zones.length === 0 ||
+        (config.cameras[search.camera].objects.genai.required_zones.length ===
+          0 ||
           search.zones.some((zone) =>
-            config.cameras[search.camera].genai.required_zones.includes(zone),
+            config.cameras[search.camera].objects.genai.required_zones.includes(
+              zone,
+            ),
           )) &&
-        (config.cameras[search.camera].genai.objects.length === 0 ||
+        (config.cameras[search.camera].objects.genai.objects.length === 0 ||
-          config.cameras[search.camera].genai.objects.includes(
+          config.cameras[search.camera].objects.genai.objects.includes(
             search.label,
           )) ? (
           <>
@@ -972,47 +975,49 @@ function ObjectDetailsTab({
         )}
 
         <div className="flex w-full flex-row justify-end gap-2">
-          {config?.cameras[search.camera].genai.enabled && search.end_time && (
+          {config?.cameras[search.camera].objects.genai.enabled &&
-            <div className="flex items-start">
+            search.end_time && (
-              <Button
+              <div className="flex items-start">
-                className="rounded-r-none border-r-0"
+                <Button
-                aria-label={t("details.button.regenerate.label")}
+                  className="rounded-r-none border-r-0"
-                onClick={() => regenerateDescription("thumbnails")}
+                  aria-label={t("details.button.regenerate.label")}
-              >
+                  onClick={() => regenerateDescription("thumbnails")}
-                {t("details.button.regenerate.title")}
+                >
-              </Button>
+                  {t("details.button.regenerate.title")}
-              {search.has_snapshot && (
+                </Button>
-                <DropdownMenu>
+                {search.has_snapshot && (
-                  <DropdownMenuTrigger asChild>
+                  <DropdownMenu>
-                    <Button
+                    <DropdownMenuTrigger asChild>
-                      className="rounded-l-none border-l-0 px-2"
+                      <Button
-                      aria-label={t("details.expandRegenerationMenu")}
+                        className="rounded-l-none border-l-0 px-2"
-                    >
+                        aria-label={t("details.expandRegenerationMenu")}
-                      <FaChevronDown className="size-3" />
+                      >
-                    </Button>
+                        <FaChevronDown className="size-3" />
-                  </DropdownMenuTrigger>
+                      </Button>
-                  <DropdownMenuContent>
+                    </DropdownMenuTrigger>
-                    <DropdownMenuItem
+                    <DropdownMenuContent>
-                      className="cursor-pointer"
+                      <DropdownMenuItem
-                      aria-label={t("details.regenerateFromSnapshot")}
+                        className="cursor-pointer"
-                      onClick={() => regenerateDescription("snapshot")}
+                        aria-label={t("details.regenerateFromSnapshot")}
-                    >
+                        onClick={() => regenerateDescription("snapshot")}
-                      {t("details.regenerateFromSnapshot")}
+                      >
-                    </DropdownMenuItem>
+                        {t("details.regenerateFromSnapshot")}
-                    <DropdownMenuItem
+                      </DropdownMenuItem>
-                      className="cursor-pointer"
+                      <DropdownMenuItem
-                      aria-label={t("details.regenerateFromThumbnails")}
+                        className="cursor-pointer"
-                      onClick={() => regenerateDescription("thumbnails")}
+                        aria-label={t("details.regenerateFromThumbnails")}
-                    >
+                        onClick={() => regenerateDescription("thumbnails")}
-                      {t("details.regenerateFromThumbnails")}
+                      >
-                    </DropdownMenuItem>
+                        {t("details.regenerateFromThumbnails")}
-                  </DropdownMenuContent>
+                      </DropdownMenuItem>
-                </DropdownMenu>
+                    </DropdownMenuContent>
-              )}
+                  </DropdownMenu>
-            </div>
+                )}
-          )}
+              </div>
-          {((config?.cameras[search.camera].genai.enabled && search.end_time) ||
+            )}
-            !config?.cameras[search.camera].genai.enabled) && (
+          {((config?.cameras[search.camera].objects.genai.enabled &&
+            search.end_time) ||
+            !config?.cameras[search.camera].objects.genai.enabled) && (
             <Button
               variant="select"
               aria-label={t("button.save", { ns: "common" })}

web/src/types/frigateConfig.ts

@@ -94,13 +94,6 @@ export interface CameraConfig {
     cmd: string;
     roles: string[];
   }[];
-  genai: {
-    enabled: string;
-    prompt: string;
-    object_prompts: { [key: string]: string };
-    required_zones: string[];
-    objects: string[];
-  };
   live: {
     height: number;
     quality: number;
@@ -146,6 +139,14 @@ export interface CameraConfig {
     };
     mask: string;
     track: string[];
+    genai: {
+      enabled: boolean;
+      enabled_in_config: boolean;
+      prompt: string;
+      object_prompts: { [key: string]: string };
+      required_zones: string[];
+      objects: string[];
+    };
   };
   onvif: {
     autotracking: {
@@ -406,15 +407,10 @@ export interface FrigateConfig {
   };
 
   genai: {
-    enabled: boolean;
     provider: string;
     base_url?: string;
     api_key?: string;
     model: string;
-    prompt: string;
-    object_prompts: { [key: string]: string };
-    required_zones: string[];
-    objects: string[];
   };
 
   go2rtc: {

web/src/views/settings/CameraSettingsView.tsx

@@ -413,7 +413,7 @@ export default function CameraSettingsView({
                   </div>
                 </div>
               </div>
-              {config?.genai?.enabled && (
+              {cameraConfig?.objects?.genai?.enabled_in_config && (
                 <>
                   <Separator className="my-2 flex bg-secondary" />