Add audio labelmap grouping (#24004)

Allow audio classes to be grouped under a shared configured label.

Keep audio overrides separate from object labels and retain only the highest-scoring grouped detection.

Refs #23967

docs/docs/configuration/advanced/reference.md

@@ -217,6 +217,8 @@ audio:
     - fire_alarm
     - speech
     - yell
+  # Optional: Audio label name modifications. These are merged into the standard audio labelmap.
+  labelmap: {}
   # Optional: Filters to configure detection.
   filters:
     # Label that matches label in listen config.

docs/docs/configuration/audio_detectors.md

@@ -114,6 +114,30 @@ audio:
 </TabItem>
 </ConfigTabs>
 
+#### Grouping Audio Labels
+
+Related audio classes can be grouped under one label by mapping their numeric
+class IDs to the same name. Add the grouped name to `listen` and use it for any
+corresponding filter:
+
+```yaml
+audio:
+  listen:
+    - dogs
+  labelmap:
+    69: dogs # dog
+    70: dogs # bark
+    75: dogs # whimper_dog
+  filters:
+    dogs:
+      threshold: 0.8
+```
+
+Class IDs are zero-based indices in
+[`audio-labelmap.txt`](https://github.com/blakeblackshear/frigate/blob/dev/audio-labelmap.txt),
+so each ID is one less than the displayed file line number.
+Audio label mappings are separate from the object detector's `model.labelmap`.
+
 ### Common Audio Labels
 
 The labelmap includes hundreds of sound types. The labels below are the ones most users may find practical, grouped by what they're typically used for. Use the exact label string from the left column in your `listen` config, or search for the label in the Frigate UI directly.

frigate/config/camera/audio.py

@@ -41,6 +41,11 @@ class AudioConfig(FrigateBaseModel):
         title="Listen types",
         description="List of audio event types to detect (for example: bark, fire_alarm, speech, yell).",
     )
+    labelmap: dict[int, str] = Field(
+        default_factory=dict,
+        title="Audio labelmap customization",
+        description="Overrides or remapping entries to merge into the standard audio labelmap.",
+    )
     filters: dict[str, AudioFilterConfig] | None = Field(
         None,
         title="Audio filters",

frigate/events/audio.py

@@ -210,7 +210,11 @@ class AudioEventMaintainer(threading.Thread):
         # per-camera stop signal so a single maintainer can be torn down at
         # runtime (e.g. on camera removal) without stopping the whole process
         self.camera_stop_event = threading.Event()
-        self.detector = AudioTfl(stop_event, self.camera_config.audio.num_threads)
+        self.detector = AudioTfl(
+            stop_event,
+            self.camera_config.audio.num_threads,
+            self.camera_config.audio.labelmap,
+        )
         self.shape = (int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE)),)
         self.chunk_size = int(round(AUDIO_DURATION * AUDIO_SAMPLE_RATE * 2))
         self.logger = logging.getLogger(f"audio.{self.camera_config.name}")
@@ -392,7 +396,10 @@ class AudioEventMaintainer(threading.Thread):
 
         while not self.stop_event.is_set() and not self.camera_stop_event.is_set():
             # check if there is an updated config
-            self.config_subscriber.check_for_updates()
+            updated_topics = self.config_subscriber.check_for_updates()
+
+            if CameraConfigUpdateEnum.audio.name in updated_topics:
+                self.detector.update_labelmap(self.camera_config.audio.labelmap)
 
             enabled = self.camera_config.enabled
             if enabled != self.was_enabled:
@@ -451,10 +458,17 @@ class AudioEventMaintainer(threading.Thread):
 
 
 class AudioTfl:
-    def __init__(self, stop_event: threading.Event, num_threads: int = 2) -> None:
+    def __init__(
+        self,
+        stop_event: threading.Event,
+        num_threads: int = 2,
+        labelmap: dict[int, str] | None = None,
+    ) -> None:
         self.stop_event = stop_event
         self.num_threads = num_threads
-        self.labels = load_labels("/audio-labelmap.txt", prefill=521)
+        self._default_labels = load_labels("/audio-labelmap.txt", prefill=521)
+        self.labels: dict[int, str] = {}
+        self.update_labelmap(labelmap or {})
         # Suppress TFLite delegate creation messages that bypass Python logging
         with suppress_stderr_during("tflite_interpreter_init"):
             self.interpreter = Interpreter(
@@ -466,6 +480,10 @@ class AudioTfl:
         self.tensor_input_details = self.interpreter.get_input_details()
         self.tensor_output_details = self.interpreter.get_output_details()
 
+    def update_labelmap(self, labelmap: dict[int, str]) -> None:
+        """Merge configured label overrides into the default audio labelmap."""
+        self.labels = {**self._default_labels, **labelmap}
+
     def _detect_raw(self, tensor_input: np.ndarray) -> np.ndarray:
         self.interpreter.set_tensor(self.tensor_input_details[0]["index"], tensor_input)
         self.interpreter.invoke()
@@ -504,10 +522,14 @@ class AudioTfl:
 
         raw_detections = self._detect_raw(tensor_input)
 
+        detected_labels: set[str] = set()
+
         for d in raw_detections:
             if d[1] < threshold:
                 break
-            detections.append(
-                (self.labels[int(d[0])], float(d[1]), (d[2], d[3], d[4], d[5]))
-            )
+            label = self.labels[int(d[0])]
+            if label in detected_labels:
+                continue
+            detected_labels.add(label)
+            detections.append((label, float(d[1]), (d[2], d[3], d[4], d[5])))
         return detections

frigate/test/test_audio.py

@@ -0,0 +1,75 @@
+"""Tests for audio label mapping."""
+
+import threading
+import unittest
+from unittest.mock import Mock
+
+import numpy as np
+
+from frigate.events.audio import AudioTfl
+
+
+class TestAudioTfl(unittest.TestCase):
+    def setUp(self):
+        self.detector = AudioTfl.__new__(AudioTfl)
+        self.detector.stop_event = threading.Event()
+        self.detector._default_labels = {
+            69: "dog",
+            70: "bark",
+            75: "whimper_dog",
+            117: "dogs",
+        }
+
+    def test_update_labelmap_replaces_and_resets_overrides(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs"})
+        assert self.detector.labels == {
+            69: "dogs",
+            70: "dogs",
+            75: "whimper_dog",
+            117: "dogs",
+        }
+
+        self.detector.update_labelmap({75: "whimper"})
+        assert self.detector.labels == {
+            69: "dog",
+            70: "bark",
+            75: "whimper",
+            117: "dogs",
+        }
+
+    def test_detect_returns_highest_scoring_detection_for_grouped_label(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"})
+        self.detector._detect_raw = Mock(
+            return_value=np.array(
+                [
+                    [117, 0.95, -1, -1, -1, -1],
+                    [70, 0.9, -1, -1, -1, -1],
+                    [69, 0.8, -1, -1, -1, -1],
+                    [75, 0.7, -1, -1, -1, -1],
+                ],
+                dtype=np.float32,
+            )
+        )
+
+        detections = self.detector.detect(np.array([], dtype=np.float32))
+
+        assert len(detections) == 1
+        assert detections[0][0] == "dogs"
+        self.assertAlmostEqual(detections[0][1], 0.95)
+
+    def test_each_dog_audio_label_maps_to_grouped_label(self):
+        self.detector.update_labelmap({69: "dogs", 70: "dogs", 75: "dogs"})
+
+        for class_id in (69, 70, 75):
+            with self.subTest(class_id=class_id):
+                self.detector._detect_raw = Mock(
+                    return_value=np.array(
+                        [[class_id, 0.9, -1, -1, -1, -1]], dtype=np.float32
+                    )
+                )
+
+                detections = self.detector.detect(np.array([], dtype=np.float32))
+
+                assert len(detections) == 1
+                assert detections[0][0] == "dogs"
+                self.assertAlmostEqual(detections[0][1], 0.9)

frigate/test/test_config.py

@@ -1154,6 +1154,28 @@ class TestConfig(unittest.TestCase):
         frigate_config = FrigateConfig(**config)
         assert frigate_config.model.merged_labelmap[7] == "truck"
 
+    def test_audio_labelmap_inheritance_is_separate_from_model_labelmap(self):
+        config = deep_merge(
+            {
+                "audio": {"labelmap": {69: "dogs", 70: "dogs"}},
+                "cameras": {
+                    "back": {
+                        "audio": {"labelmap": {75: "dogs"}},
+                    }
+                },
+            },
+            self.minimal,
+        )
+
+        frigate_config = FrigateConfig(**config)
+
+        assert frigate_config.cameras["back"].audio.labelmap == {
+            69: "dogs",
+            70: "dogs",
+            75: "dogs",
+        }
+        assert frigate_config.model.merged_labelmap[69] != "dogs"
+
     def test_default_labelmap_empty(self):
         config = {
             "mqtt": {"host": "mqtt"},

web/public/locales/en/config/cameras.json

@@ -31,6 +31,10 @@
       "label": "Listen types",
       "description": "List of audio event types to detect (for example: bark, fire_alarm, speech, yell)."
     },
+    "labelmap": {
+      "label": "Audio labelmap customization",
+      "description": "Overrides or remapping entries to merge into the standard audio labelmap."
+    },
     "filters": {
       "label": "Audio filters",
       "description": "Per-audio-type filter settings such as confidence thresholds used to reduce false positives.",

web/public/locales/en/config/global.json

@@ -533,6 +533,10 @@
       "label": "Listen types",
       "description": "List of audio event types to detect (for example: bark, fire_alarm, speech, yell)."
     },
+    "labelmap": {
+      "label": "Audio labelmap customization",
+      "description": "Overrides or remapping entries to merge into the standard audio labelmap."
+    },
     "filters": {
       "label": "Audio filters",
       "description": "Per-audio-type filter settings such as confidence thresholds used to reduce false positives.",

web/src/components/config-form/section-configs/audio.ts

@@ -31,7 +31,7 @@ const audio: SectionConfigOverrides = {
       detection: ["listen", "filters"],
       sensitivity: ["min_volume", "max_not_heard"],
     },
-    hiddenFields: ["enabled_in_config"],
+    hiddenFields: ["enabled_in_config", "labelmap"],
     advancedFields: ["min_volume", "max_not_heard", "num_threads"],
     uiSchema: {
       filters: {