nik/frigate
1
0
Fork 0
mirror of https://github.com/blakeblackshear/frigate.git synced 2026-08-31 07:27:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Don't require a restart to enable GenAI descriptions (#23964)

Browse Source 
* create GenAI post processors when a camera enables GenAI at runtime

* fix types
This commit is contained in:
Josh Hawkins 2026-08-12 08:55:34 -05:00 committed by GitHub
parent c75611b4df
commit aff9799451
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 285 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
frigate/app.py
View File

@ -103,21 +103,9 @@ class FrigateApp:
self.detection_shms: list[mp.shared_memory.SharedMemory] = []
self.log_queue: Queue = mp.Queue()
self.camera_metrics: DictProxy = self.metrics_manager.dict()
self.embeddings_metrics: DataProcessorMetrics | None = (
DataProcessorMetrics(
self.metrics_manager, list(config.classification.custom.keys())
)
if (
config.semantic_search.enabled
or any(
c.objects.genai.enabled or c.review.genai.enabled
for c in config.cameras.values()
)
or config.lpr.enabled
or config.face_recognition.enabled
or len(config.classification.custom) > 0
)
else None
self.embeddings_metrics = DataProcessorMetrics(
self.metrics_manager, list(config.classification.custom.keys())
)
self.ptz_metrics: dict[str, PTZMetrics] = {}
self.processes: dict[str, int] = {}

6
frigate/data_processing/post/object_descriptions.py
View File

@ -63,8 +63,10 @@ class ObjectDescriptionProcessor(PostProcessorApi):
"""Handle an update to a frame for an object."""
camera_config = self.config.cameras[camera]
# no need to save our own thumbnails if genai is not enabled
# or if the object has become stationary
if not camera_config.objects.genai.enabled:
return
# no need to save our own thumbnails if the object has become stationary
if not data["stationary"]:
if data["id"] not in self.tracked_events:
self.tracked_events[data["id"]] = []

2
frigate/embeddings/__init__.py
View File

@ -33,7 +33,7 @@ class EmbeddingProcess(FrigateProcess):
def __init__(
self,
config: FrigateConfig,
metrics: DataProcessorMetrics | None,
metrics: DataProcessorMetrics,
stop_event: MpEvent,
) -> None:
super().__init__(

82
frigate/embeddings/maintainer.py
View File

@ -78,6 +78,16 @@ logger = logging.getLogger(__name__)
MAX_THUMBNAILS = 10
GENAI_UPDATE_TOPICS = frozenset(
{
CameraConfigUpdateEnum.add.name,
CameraConfigUpdateEnum.objects.name,
CameraConfigUpdateEnum.object_genai.name,
CameraConfigUpdateEnum.review.name,
CameraConfigUpdateEnum.review_genai.name,
}
)
class EmbeddingMaintainer(threading.Thread):
"""Handle embedding queue and post event updates."""
@ -85,7 +95,7 @@ class EmbeddingMaintainer(threading.Thread):
def __init__(
self,
config: FrigateConfig,
metrics: DataProcessorMetrics | None,
metrics: DataProcessorMetrics,
stop_event: MpEvent,
) -> None:
super().__init__(name="embeddings_maintainer")
@ -220,16 +230,6 @@ class EmbeddingMaintainer(threading.Thread):
# post processors
self.post_processors: list[PostProcessorApi] = []
if any(c.review.genai.enabled_in_config for c in self.config.cameras.values()):
self.post_processors.append(
ReviewDescriptionProcessor(
self.config,
self.requestor,
self.metrics,
self.genai_manager,
)
)
if self.config.lpr.enabled:
self.post_processors.append(
LicensePlatePostProcessor(
@ -252,9 +252,9 @@ class EmbeddingMaintainer(threading.Thread):
)
)
semantic_trigger_processor: SemanticTriggerProcessor | None = None
self.semantic_trigger_processor: SemanticTriggerProcessor | None = None
if self.config.semantic_search.enabled:
semantic_trigger_processor = SemanticTriggerProcessor(
self.semantic_trigger_processor = SemanticTriggerProcessor(
db,
self.config,
self.requestor,
@ -262,9 +262,49 @@ class EmbeddingMaintainer(threading.Thread):
metrics,
self.embeddings,
)
self.post_processors.append(semantic_trigger_processor)
self.post_processors.append(self.semantic_trigger_processor)
if any(c.objects.genai.enabled_in_config for c in self.config.cameras.values()):
self._sync_genai_processors()
self.stop_event = stop_event
# recordings data
self.recordings_available_through: dict[str, float] = {}
def _sync_genai_processors(self) -> None:
"""Create GenAI post processors for cameras that have GenAI enabled.
Called at startup and again after camera config updates so enabling
GenAI on the first camera does not require a restart. Processors are
never removed once created.
A profile can turn GenAI on without setting enabled_in_config, so both
flags are checked.
"""
cameras = self.config.cameras.values()
if any(
c.review.genai.enabled or c.review.genai.enabled_in_config for c in cameras
) and not any(
isinstance(p, ReviewDescriptionProcessor) for p in self.post_processors
):
logger.debug("Initializing review description processor")
self.post_processors.append(
ReviewDescriptionProcessor(
self.config,
self.requestor,
self.metrics,
self.genai_manager,
)
)
if any(
c.objects.genai.enabled or c.objects.genai.enabled_in_config
for c in cameras
) and not any(
isinstance(p, ObjectDescriptionProcessor) for p in self.post_processors
):
logger.debug("Initializing object description processor")
self.post_processors.append(
ObjectDescriptionProcessor(
self.config,
@ -272,19 +312,21 @@ class EmbeddingMaintainer(threading.Thread):
self.requestor,
self.metrics,
self.genai_manager,
semantic_trigger_processor,
self.semantic_trigger_processor,
)
)
self.stop_event = stop_event
def _check_camera_config_updates(self) -> None:
"""Apply camera config updates and register newly enabled processors."""
updated_topics = self.config_updater.check_for_updates()
# recordings data
self.recordings_available_through: dict[str, float] = {}
if updated_topics.keys() & GENAI_UPDATE_TOPICS:
self._sync_genai_processors()
def run(self) -> None:
"""Maintain a SQLite-vec database for semantic search."""
while not self.stop_event.is_set():
self.config_updater.check_for_updates()
self._check_camera_config_updates()
self._check_enrichment_config_updates()
self._process_requests()
self._process_updates()

2
frigate/stats/util.py
View File

@ -62,7 +62,7 @@ def get_latest_version(config: FrigateConfig) -> str:
def stats_init(
config: FrigateConfig,
camera_metrics: DictProxy,
embeddings_metrics: DataProcessorMetrics | None,
embeddings_metrics: DataProcessorMetrics,
detectors: dict[str, ObjectDetectProcess],
processes: dict[str, int],
) -> StatsTrackingTypes:

213
frigate/test/test_genai_processor_sync.py Normal file
View File

@ -0,0 +1,213 @@
"""Tests for GenAI enablement gating in the embeddings maintainer.
Covers creating post processors when GenAI is enabled at runtime, and the
per-camera gating those processors apply once they exist.
"""
import sys
import unittest
from unittest.mock import MagicMock, patch
# Mock TFLite before importing the maintainer
_MOCK_MODULES = [
"tflite_runtime",
"tflite_runtime.interpreter",
"ai_edge_litert",
"ai_edge_litert.interpreter",
]
for mod in _MOCK_MODULES:
if mod not in sys.modules:
sys.modules[mod] = MagicMock()
# imported from the maintainer to avoid tripping the circular import between
# the maintainer and the processor modules
from frigate.embeddings.maintainer import ( # noqa: E402
EmbeddingMaintainer,
ObjectDescriptionProcessor,
PostProcessDataEnum,
ReviewDescriptionProcessor,
)
class TestGenAIProcessorSync(unittest.TestCase):
"""Enabling GenAI on the first camera must not require a restart."""
def _make_maintainer(
self,
review: bool = False,
objects: bool = False,
review_in_config: bool | None = None,
objects_in_config: bool | None = None,
) -> EmbeddingMaintainer:
# Bypass the heavy __init__; only the attributes touched by
# _sync_genai_processors are needed for these tests.
maintainer = EmbeddingMaintainer.__new__(EmbeddingMaintainer)
maintainer.post_processors = []
maintainer.config = MagicMock()
maintainer.config.cameras = {
"front": self._make_camera(
review,
objects,
review if review_in_config is None else review_in_config,
objects if objects_in_config is None else objects_in_config,
)
}
maintainer.config_updater = MagicMock()
maintainer.embeddings = None
maintainer.requestor = MagicMock()
maintainer.metrics = MagicMock()
maintainer.genai_manager = MagicMock()
maintainer.semantic_trigger_processor = None
return maintainer
def _make_camera(
self,
review: bool,
objects: bool,
review_in_config: bool,
objects_in_config: bool,
) -> MagicMock:
camera = MagicMock()
camera.review.genai.enabled = review
camera.review.genai.enabled_in_config = review_in_config
camera.objects.genai.enabled = objects
camera.objects.genai.enabled_in_config = objects_in_config
return camera
def _processor_types(self, maintainer: EmbeddingMaintainer) -> list[type]:
return [type(p) for p in maintainer.post_processors]
def test_no_processors_when_genai_disabled(self):
"""A config with no GenAI cameras registers neither processor."""
maintainer = self._make_maintainer()
maintainer._sync_genai_processors()
self.assertEqual(maintainer.post_processors, [])
def test_review_processor_added_when_enabled_after_startup(self):
"""Enabling review GenAI on the first camera registers the processor."""
maintainer = self._make_maintainer()
maintainer._sync_genai_processors()
camera = maintainer.config.cameras["front"]
camera.review.genai.enabled = True
camera.review.genai.enabled_in_config = True
maintainer._sync_genai_processors()
self.assertEqual(
self._processor_types(maintainer), [ReviewDescriptionProcessor]
)
def test_object_processor_added_when_enabled_after_startup(self):
"""Enabling object GenAI on the first camera registers the processor."""
maintainer = self._make_maintainer()
maintainer._sync_genai_processors()
camera = maintainer.config.cameras["front"]
camera.objects.genai.enabled = True
camera.objects.genai.enabled_in_config = True
maintainer._sync_genai_processors()
self.assertEqual(
self._processor_types(maintainer), [ObjectDescriptionProcessor]
)
def test_processor_added_when_only_enabled_by_profile(self):
"""A profile enables GenAI without setting enabled_in_config."""
maintainer = self._make_maintainer(
review=True, objects=True, review_in_config=False, objects_in_config=False
)
maintainer._sync_genai_processors()
self.assertEqual(
self._processor_types(maintainer),
[ReviewDescriptionProcessor, ObjectDescriptionProcessor],
)
def test_processors_are_not_duplicated(self):
"""Repeated config updates must not register a second processor."""
maintainer = self._make_maintainer(review=True, objects=True)
maintainer._sync_genai_processors()
maintainer._sync_genai_processors()
self.assertEqual(
self._processor_types(maintainer),
[ReviewDescriptionProcessor, ObjectDescriptionProcessor],
)
def test_genai_topic_triggers_sync(self):
"""A camera config update on a GenAI topic registers the processor."""
maintainer = self._make_maintainer(review=True)
maintainer.config_updater.check_for_updates.return_value = {"review": ["front"]}
maintainer._check_camera_config_updates()
self.assertEqual(
self._processor_types(maintainer), [ReviewDescriptionProcessor]
)
def test_unrelated_topic_does_not_sync(self):
"""An unrelated camera config update must not register processors."""
maintainer = self._make_maintainer(review=True)
maintainer.config_updater.check_for_updates.return_value = {"motion": ["front"]}
maintainer._check_camera_config_updates()
self.assertEqual(maintainer.post_processors, [])
class TestObjectDescriptionCameraGating(unittest.TestCase):
"""One camera enabling object descriptions must not enlist the others."""
def _make_processor(self, enabled: bool) -> ObjectDescriptionProcessor:
config = MagicMock()
camera = MagicMock()
camera.objects.genai.enabled = enabled
camera.objects.genai.send_triggers.after_significant_updates = None
config.cameras = {"front": camera}
genai_manager = MagicMock()
genai_manager.description_client = MagicMock()
return ObjectDescriptionProcessor(
config, None, MagicMock(), MagicMock(), genai_manager, None
)
def _update(self, processor: ObjectDescriptionProcessor) -> None:
processor.process_data(
{
"camera": "front",
"data": {
"id": "1234.5-abcdef",
"box": (0, 0, 10, 10),
"stationary": False,
},
"state": "update",
"yuv_frame": MagicMock(),
},
PostProcessDataEnum.tracked_object,
)
@patch("frigate.data_processing.post.object_descriptions.create_thumbnail")
def test_disabled_camera_collects_no_thumbnails(self, mock_create_thumbnail):
"""A camera with object descriptions off does no thumbnail work."""
processor = self._make_processor(enabled=False)
self._update(processor)
mock_create_thumbnail.assert_not_called()
self.assertEqual(processor.tracked_events, {})
@patch("frigate.data_processing.post.object_descriptions.create_thumbnail")
def test_enabled_camera_collects_thumbnails(self, mock_create_thumbnail):
"""A camera with object descriptions on still collects thumbnails."""
mock_create_thumbnail.return_value = b"jpg"
processor = self._make_processor(enabled=True)
self._update(processor)
mock_create_thumbnail.assert_called_once()
self.assertEqual(len(processor.tracked_events["1234.5-abcdef"]), 1)

2
frigate/types.py
View File

@ -8,7 +8,7 @@ from frigate.object_detection.base import ObjectDetectProcess
class StatsTrackingTypes(TypedDict):
camera_metrics: dict[str, CameraMetrics]
embeddings_metrics: DataProcessorMetrics | None
embeddings_metrics: DataProcessorMetrics
detectors: dict[str, ObjectDetectProcess]
started: int
latest_frigate_version: str