diff --git a/docs/docs/configuration/advanced/reference.md b/docs/docs/configuration/advanced/reference.md index cf601b1dd7..856f70c865 100644 --- a/docs/docs/configuration/advanced/reference.md +++ b/docs/docs/configuration/advanced/reference.md @@ -217,6 +217,8 @@ audio: - fire_alarm - speech - yell + # Optional: Audio label name modifications. These are merged into the standard audio labelmap. + labelmap: {} # Optional: Filters to configure detection. filters: # Label that matches label in listen config. diff --git a/docs/docs/configuration/audio_detectors.md b/docs/docs/configuration/audio_detectors.md index 39ec2e9ff9..dba42135b4 100644 --- a/docs/docs/configuration/audio_detectors.md +++ b/docs/docs/configuration/audio_detectors.md @@ -114,6 +114,30 @@ audio: +#### Grouping Audio Labels + +Related audio classes can be grouped under one label by mapping their numeric +class IDs to the same name. Add the grouped name to `listen` and use it for any +corresponding filter: + +```yaml +audio: + listen: + - dogs + labelmap: + 69: dogs # dog + 70: dogs # bark + 75: dogs # whimper_dog + filters: + dogs: + threshold: 0.8 +``` + +Class IDs are zero-based indices in +[`audio-labelmap.txt`](https://github.com/blakeblackshear/frigate/blob/dev/audio-labelmap.txt), +so each ID is one less than the displayed file line number. +Audio label mappings are separate from the object detector's `model.labelmap`. + ### Common Audio Labels The labelmap includes hundreds of sound types. The labels below are the ones most users may find practical, grouped by what they're typically used for. Use the exact label string from the left column in your `listen` config, or search for the label in the Frigate UI directly. diff --git a/frigate/config/camera/audio.py b/frigate/config/camera/audio.py index 813c2988db..37114e3ae5 100644 --- a/frigate/config/camera/audio.py +++ b/frigate/config/camera/audio.py @@ -41,6 +41,11 @@ class AudioConfig(FrigateBaseModel): title="Listen types", description="List of audio event types to detect (for example: bark, fire_alarm, speech, yell).", ) + labelmap: dict[int, str] = Field( + default_factory=dict, + title="Audio labelmap customization", + description="Overrides or remapping entries to merge into the standard audio labelmap.", + ) filters: dict[str, AudioFilterConfig] | None = Field( None, title="Audio filters", diff --git a/frigate/events/audio.py b/frigate/events/audio.py index 9af1bd26b2..4d7c96a347 100644 --- a/frigate/events/audio.py +++ b/frigate/events/audio.py @@ -210,7 +210,11 @@ class AudioEventMaintainer(threading.Thread): # per-camera stop signal so a single maintainer can be torn down at # runtime (e.g. on camera removal) without stopping the whole process self.camera_stop_event = threading.Event() - self.detector = AudioTfl(stop_event, self.camera_config.audio.num_threads) + self.detector = AudioTfl( + stop_event, + self.camera_config.audio.num_threads, + self.camera_config.audio.labelmap, + ) self.shape = (int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE)),) self.chunk_size = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE * 2)) self.logger = logging.getLogger(f"audio.{self.camera_config.name}") @@ -392,7 +396,10 @@ class AudioEventMaintainer(threading.Thread): while not self.stop_event.is_set() and not self.camera_stop_event.is_set(): # check if there is an updated config - self.config_subscriber.check_for_updates() + updated_topics = self.config_subscriber.check_for_updates() + + if CameraConfigUpdateEnum.audio.name in updated_topics: + self.detector.update_labelmap(self.camera_config.audio.labelmap) enabled = self.camera_config.enabled if enabled != self.was_enabled: @@ -451,10 +458,17 @@ class AudioEventMaintainer(threading.Thread): class AudioTfl: - def __init__(self, stop_event: threading.Event, num_threads: int = 2) -> None: + def __init__( + self, + stop_event: threading.Event, + num_threads: int = 2, + labelmap: dict[int, str] | None = None, + ) -> None: self.stop_event = stop_event self.num_threads = num_threads - self.labels = load_labels("/audio-labelmap.txt", prefill=521) + self._default_labels = load_labels("/audio-labelmap.txt", prefill=521) + self.labels: dict[int, str] = {} + self.update_labelmap(labelmap or {}) # Suppress TFLite delegate creation messages that bypass Python logging with suppress_stderr_during("tflite_interpreter_init"): self.interpreter = Interpreter( @@ -466,6 +480,10 @@ class AudioTfl: self.tensor_input_details = self.interpreter.get_input_details() self.tensor_output_details = self.interpreter.get_output_details() + def update_labelmap(self, labelmap: dict[int, str]) -> None: + """Merge configured label overrides into the default audio labelmap.""" + self.labels = {**self._default_labels, **labelmap} + def _detect_raw(self, tensor_input: np.ndarray) -> np.ndarray: self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input) self.interpreter.invoke() @@ -504,10 +522,14 @@ class AudioTfl: raw_detections = self._detect_raw(tensor_input) + detected_labels: set[str] = set() + for d in raw_detections: if d[1] < threshold: break - detections.append( - (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5])) - ) + label = self.labels[int(d[0])] + if label in detected_labels: + continue + detected_labels.add(label) + detections.append((label, float(d[1]), (d[2], d[3], d[4], d[5]))) return detections diff --git a/frigate/test/test_audio.py b/frigate/test/test_audio.py new file mode 100644 index 0000000000..dc231c6dea --- /dev/null +++ b/frigate/test/test_audio.py @@ -0,0 +1,75 @@ +"""Tests for audio label mapping.""" + +import threading +import unittest +from unittest.mock import Mock + +import numpy as np + +from frigate.events.audio import AudioTfl + + +class TestAudioTfl(unittest.TestCase): + def setUp(self): + self.detector = AudioTfl.__new__(AudioTfl) + self.detector.stop_event = threading.Event() + self.detector._default_labels = { + 69: "dog", + 70: "bark", + 75: "whimper_dog", + 117: "dogs", + } + + def test_update_labelmap_replaces_and_resets_overrides(self): + self.detector.update_labelmap({69: "dogs", 70: "dogs"}) + assert self.detector.labels == { + 69: "dogs", + 70: "dogs", + 75: "whimper_dog", + 117: "dogs", + } + + self.detector.update_labelmap({75: "whimper"}) + assert self.detector.labels == { + 69: "dog", + 70: "bark", + 75: "whimper", + 117: "dogs", + } + + def test_detect_returns_highest_scoring_detection_for_grouped_label(self): + self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"}) + self.detector._detect_raw = Mock( + return_value=np.array( + [ + [117, 0.95, -1, -1, -1, -1], + [70, 0.9, -1, -1, -1, -1], + [69, 0.8, -1, -1, -1, -1], + [75, 0.7, -1, -1, -1, -1], + ], + dtype=np.float32, + ) + ) + + detections = self.detector.detect(np.array([], dtype=np.float32)) + + assert len(detections) == 1 + assert detections[0][0] == "dogs" + self.assertAlmostEqual(detections[0][1], 0.95) + + def test_each_dog_audio_label_maps_to_grouped_label(self): + self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"}) + + for class_id in (69, 70, 75): + with self.subTest(class_id=class_id): + self.detector._detect_raw = Mock( + return_value=np.array( + [[class_id, 0.9, -1, -1, -1, -1]], dtype=np.float32 + ) + ) + + detections = self.detector.detect(np.array([], dtype=np.float32)) + + assert len(detections) == 1 + assert detections[0][0] == "dogs" + self.assertAlmostEqual(detections[0][1], 0.9) diff --git a/frigate/test/test_config.py b/frigate/test/test_config.py index 585064a761..bf8be11843 100644 --- a/frigate/test/test_config.py +++ b/frigate/test/test_config.py @@ -1154,6 +1154,28 @@ class TestConfig(unittest.TestCase): frigate_config = FrigateConfig(**config) assert frigate_config.model.merged_labelmap[7] == "truck" + def test_audio_labelmap_inheritance_is_separate_from_model_labelmap(self): + config = deep_merge( + { + "audio": {"labelmap": {69: "dogs", 70: "dogs"}}, + "cameras": { + "back": { + "audio": {"labelmap": {75: "dogs"}}, + } + }, + }, + self.minimal, + ) + + frigate_config = FrigateConfig(**config) + + assert frigate_config.cameras["back"].audio.labelmap == { + 69: "dogs", + 70: "dogs", + 75: "dogs", + } + assert frigate_config.model.merged_labelmap[69] != "dogs" + def test_default_labelmap_empty(self): config = { "mqtt": {"host": "mqtt"}, diff --git a/web/public/locales/en/config/cameras.json b/web/public/locales/en/config/cameras.json index ad07847cdd..5bf89725ac 100644 --- a/web/public/locales/en/config/cameras.json +++ b/web/public/locales/en/config/cameras.json @@ -31,6 +31,10 @@ "label": "Listen types", "description": "List of audio event types to detect (for example: bark, fire_alarm, speech, yell)." }, + "labelmap": { + "label": "Audio labelmap customization", + "description": "Overrides or remapping entries to merge into the standard audio labelmap." + }, "filters": { "label": "Audio filters", "description": "Per-audio-type filter settings such as confidence thresholds used to reduce false positives.", diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index 3a7fe00c4c..c4755a61fe 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -533,6 +533,10 @@ "label": "Listen types", "description": "List of audio event types to detect (for example: bark, fire_alarm, speech, yell)." }, + "labelmap": { + "label": "Audio labelmap customization", + "description": "Overrides or remapping entries to merge into the standard audio labelmap." + }, "filters": { "label": "Audio filters", "description": "Per-audio-type filter settings such as confidence thresholds used to reduce false positives.", diff --git a/web/src/components/config-form/section-configs/audio.ts b/web/src/components/config-form/section-configs/audio.ts index dda6198c52..06f1670c67 100644 --- a/web/src/components/config-form/section-configs/audio.ts +++ b/web/src/components/config-form/section-configs/audio.ts @@ -31,7 +31,7 @@ const audio: SectionConfigOverrides = { detection: ["listen", "filters"], sensitivity: ["min_volume", "max_not_heard"], }, - hiddenFields: ["enabled_in_config"], + hiddenFields: ["enabled_in_config", "labelmap"], advancedFields: ["min_volume", "max_not_heard", "num_threads"], uiSchema: { filters: {