enhance scale count resolution logic

frigate/detectors/plugins/deepx.py

@@ -63,9 +63,9 @@ PPU_ANCHORS_BY_SCALES = {
 }
 
 PPU_MAX_STRIDE = 32
-# the largest scale count we have a table for; below this, a resolved
-# scale_count is under-evidenced rather than disproven, so it still gets
-# the biggest table as a best guess
+# the largest scale count we have a table for; a count below this with no
+# table of its own is under-evidenced rather than disproven, so it still
+# gets the biggest table as a best guess
 PPU_MAX_KNOWN_SCALES = max(PPU_ANCHORS_BY_SCALES)
 
 # A nms-in-head output is (N, 6) rows of x_min, y_min, x_max, y_max, score,
@@ -429,15 +429,39 @@ def ppu_needs_grid_decode(records: np.ndarray) -> bool:
     return len(np.unique(layer_idx)) > 1
 
 
-def resolve_scale_count(records: np.ndarray, scale_count: int | None) -> int:
-    """`scale_count` if given, else the layer count this frame alone proves,
-    defaulting to 3 when the frame has no records to read it from."""
+def ppu_scale_count_lower_bound(records: np.ndarray, width: int, height: int) -> int:
+    """The fewest detection scales these records can come from. Each proves
+    one past its layer index, and its grid cell can prove more: cell g of
+    layer L only exists if that layer's stride keeps stride * g < input,
+    and the stride is PPU_MAX_STRIDE >> (scales - 1 - L), so a cell too
+    far right or down for a coarse stride forces finer strides below it.
+    Only a lower bound: nothing in a record rules out scales above it."""
+    grid = reinterpret(records, PPU_GRID_BYTES, np.uint8)
+    grid_y = grid[:, 0].astype(np.int64)
+    grid_x = grid[:, 1].astype(np.int64)
+    layer_idx = grid[:, 3].astype(np.int64)
+
+    # strides from the coarsest down; each one a cell overflows is one more
+    # halving the layer needs, i.e. one more scale below it
+    strides = PPU_MAX_STRIDE >> np.arange(PPU_MAX_STRIDE.bit_length())
+    overflow_x = (strides[None, :] * grid_x[:, None] >= width).sum(axis=1)
+    overflow_y = (strides[None, :] * grid_y[:, None] >= height).sum(axis=1)
+
+    return int((layer_idx + 1 + np.maximum(overflow_x, overflow_y)).max())
+
+
+def resolve_scale_count(
+    records: np.ndarray, scale_count: int | None, width: int, height: int
+) -> int:
+    """`scale_count` if given, else the fewest scales this frame alone
+    proves, defaulting to 3 when the frame has no records to read."""
     if scale_count is not None:
         return scale_count
 
-    layer_idx = reinterpret(records, PPU_GRID_BYTES, np.uint8)[:, 3]
+    if len(records) == 0:
+        return 3
 
-    return int(layer_idx.max()) + 1 if len(layer_idx) else 3
+    return ppu_scale_count_lower_bound(records, width, height)
 
 
 def ppu_grid_regression_geometry(
@@ -538,7 +562,7 @@ def decode_ppu(
     labels = reinterpret(records, PPU_LABEL_BYTES, np.uint32).flatten()
 
     if anchor_based:
-        scale_count = resolve_scale_count(records, scale_count)
+        scale_count = resolve_scale_count(records, scale_count, width, height)
         centre_x, centre_y, box_w, box_h, known = ppu_anchor_geometry(
             records, boxes, scale_count
         )
@@ -549,7 +573,7 @@ def decode_ppu(
             needs_grid_decode = ppu_needs_grid_decode(records)
 
         if needs_grid_decode:
-            scale_count = resolve_scale_count(records, scale_count)
+            scale_count = resolve_scale_count(records, scale_count, width, height)
             centre_x, centre_y, box_w, box_h, known = ppu_grid_regression_geometry(
                 records, boxes, scale_count
             )
@@ -891,7 +915,11 @@ class DeepxDetector(DetectionApi):
     def decode_ppu(self, outputs: list[np.ndarray]) -> np.ndarray:
         """Decode PPU records, keeping each heuristic once a frame settles
         it: anchor-free, needs-grid-decode, and scale count all only
-        move toward more certainty, never back."""
+        move toward more certainty, never back. The scale count is the
+        largest lower bound any frame has proven, so a three-scale head
+        whose frames so far only held its finer layers near the origin
+        reads against the two-scale table until a record lands far enough
+        right or down, or on the coarsest layer, to prove the third."""
         if self.ppu_anchor_free:
             anchor_based = False
         else:
@@ -900,8 +928,10 @@ class DeepxDetector(DetectionApi):
 
         records = ppu_records(outputs)
         if records is not None and len(records):
-            layer_idx = reinterpret(records, PPU_GRID_BYTES, np.uint8)[:, 3]
-            self.ppu_scale_count = max(self.ppu_scale_count, int(layer_idx.max()) + 1)
+            self.ppu_scale_count = max(
+                self.ppu_scale_count,
+                ppu_scale_count_lower_bound(records, self.width, self.height),
+            )
 
             if not anchor_based and not self.ppu_needs_grid_decode:
                 self.ppu_needs_grid_decode = ppu_needs_grid_decode(records)

frigate/test/test_deepx.py

@@ -24,6 +24,7 @@ from frigate.detectors.plugins.deepx import (
     decode_raw_anchor,
     decode_raw_nms_in_head,
     infer_yolo_layout,
+    ppu_scale_count_lower_bound,
     resolve_device,
     rows_with_columns,
     validate_damoyolo_outputs,
@@ -189,6 +190,73 @@ class TestDeepxPpuDecode(unittest.TestCase):
         self.assertAlmostEqual(detections[0][4], 0.137188, places=5)
         self.assertAlmostEqual(detections[0][5], 0.125750, places=5)
 
+    def test_a_grid_cell_past_the_coarser_layout_proves_a_third_scale(self):
+        """A layer-0 cell at column 40 of a 640 input needs stride 8, so the
+        head has three scales even though no layer-2 record has shown up;
+        the layer-1 record in the same frame must then read against the
+        three-scale table, not the two-scale one its layer index alone
+        would suggest."""
+        detections = decode_ppu(
+            [
+                build_ppu_records(
+                    build_ppu_record((0.5, 0.5, 0.4, 0.4), grid=(3, 4, 0, 1)),
+                    build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 40, 1, 0)),
+                )
+            ],
+            640,
+            640,
+            0.25,
+            0.45,
+        )
+
+        # layer 1 of 3 is stride 16 and box 0 is the 30x61 anchor, so the
+        # centre is (4.5, 3.5) cells out and the box is 19.2 x 39.04
+        layer_one = detections[np.isclose(detections[:, 3], 0.0975)]
+        self.assertEqual(len(layer_one), 1)
+        self.assertAlmostEqual(layer_one[0][2], 0.057, places=5)
+        self.assertAlmostEqual(layer_one[0][4], 0.118, places=5)
+        self.assertAlmostEqual(layer_one[0][5], 0.1275, places=5)
+
+    def test_scale_count_lower_bound_reads_layer_and_grid_position(self):
+        """Every record proves at least one scale past its layer, and a
+        grid cell that does not fit a coarser stride proves more: a cell
+        index g at layer L needs stride * g < input, and the stride of
+        layer L in an S-scale head is 32 >> (S - 1 - L)."""
+        cases = (
+            # layer 0 cell 19 fits stride 32 (20 cells), so one scale suffices
+            ((7, 19, 0, 0), 640, 640, 1),
+            # cell 20 needs stride 16, which is layer 0 of a two-scale head
+            ((7, 20, 0, 0), 640, 640, 2),
+            # cell 39 still fits stride 16 (40 cells)
+            ((7, 39, 0, 0), 640, 640, 2),
+            # cell 40 needs stride 8, which is layer 0 of a three-scale head
+            ((7, 40, 0, 0), 640, 640, 3),
+            # layer 1 near the origin proves only two scales
+            ((3, 4, 0, 1), 640, 640, 2),
+            # layer 1 at cell 20 needs stride 16, so three scales
+            ((3, 20, 0, 1), 640, 640, 3),
+            # rows count against the height, here 416: cell 26 needs stride 8
+            ((26, 0, 0, 0), 640, 416, 3),
+            # layer 2 alone proves three scales without any grid evidence
+            ((0, 0, 0, 2), 640, 640, 3),
+        )
+        for grid, width, height, expected in cases:
+            with self.subTest(grid=grid, width=width, height=height):
+                records = build_ppu_record((0.5, 0.5, 0.4, 0.4), grid=grid)[0]
+                self.assertEqual(
+                    ppu_scale_count_lower_bound(records, width, height), expected
+                )
+
+    def test_scale_count_lower_bound_takes_the_strongest_record(self):
+        """Records that prove different counts settle on the largest."""
+        records = build_ppu_records(
+            build_ppu_record((0.5, 0.5, 0.4, 0.4), grid=(3, 4, 0, 1)),
+            build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 40, 1, 0)),
+            build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(1, 1, 0, 0)),
+        )[0]
+
+        self.assertEqual(ppu_scale_count_lower_bound(records, 640, 640), 3)
+
     def test_anchor_based_records_with_an_unsupported_scale_count_are_dropped(self):
         """A scale_count with no anchor table must drop the records rather
         than guess against the wrong table."""
@@ -897,8 +965,9 @@ class TestDeepxDetectorLayout(unittest.TestCase):
     def test_ppu_scale_count_is_kept_once_a_coarser_layer_is_seen(self):
         """A frame that only shows layer 0 cannot tell a two-scale head from
         a three-scale one and falls back to three; once a frame proves
-        layer 1 exists, that head has only two scales and every later
-        layer-0-only frame must use the two-scale table instead."""
+        layer 1 exists, two scales is the best-evidenced count and every
+        later layer-0-only frame must use the two-scale table, until some
+        record proves a third scale."""
         detector, _ = self._detector(
             ModelTypeEnum.yologeneric, [{"shape": [8400]}], ppu=True
         )
@@ -927,6 +996,35 @@ class TestDeepxDetectorLayout(unittest.TestCase):
         # two-scale table proven by the second frame: stride 16, anchor 37x58
         self.assertAlmostEqual(third[0][3], 0.232094, places=5)
 
+    def test_ppu_scale_count_grows_when_a_grid_cell_proves_a_third_scale(self):
+        """A three-scale head whose early frames only ever hold layers 0
+        and 1 near the origin reads as two scales; a later layer-0 record
+        at column 40 of 640 needs stride 8, which proves the third scale
+        without a layer-2 record ever appearing, and every frame after
+        that must read against the three-scale table."""
+        detector, _ = self._detector(
+            ModelTypeEnum.yologeneric, [{"shape": [8400]}], ppu=True
+        )
+        detector.width, detector.height = 640, 640
+
+        layer_one = build_ppu_record((0.5, 0.5, 0.4, 0.4), grid=(3, 4, 0, 1))
+        detector.session.run.return_value = [layer_one]
+        first = detector.detect_raw(np.zeros((1, 640, 640, 3), np.uint8))
+        # read as layer 1 of 2: stride 32, anchor 81x82
+        self.assertAlmostEqual(first[0][3], 0.1845, places=5)
+        self.assertEqual(detector.ppu_scale_count, 2)
+
+        detector.session.run.return_value = [
+            build_ppu_record((0.6, 0.4, 0.3, 0.7), grid=(7, 40, 1, 0))
+        ]
+        detector.detect_raw(np.zeros((1, 640, 640, 3), np.uint8))
+        self.assertEqual(detector.ppu_scale_count, 3)
+
+        detector.session.run.return_value = [layer_one]
+        third = detector.detect_raw(np.zeros((1, 640, 640, 3), np.uint8))
+        # the same record, now layer 1 of 3: stride 16, anchor 30x61
+        self.assertAlmostEqual(third[0][3], 0.0975, places=5)
+
     def test_needs_grid_decode_is_kept_once_proven(self):
         """A frame proving a YOLOX-style grid-relative head must not fall
         back to reading raw pixels just because a later frame holds only