diff --git a/docs/docs/configuration/object_detectors.md b/docs/docs/configuration/object_detectors.md index 046c6363f1..eb7244d9ae 100644 --- a/docs/docs/configuration/object_detectors.md +++ b/docs/docs/configuration/object_detectors.md @@ -799,14 +799,10 @@ See the [installation docs](../frigate/installation.md#axelera-metis) for inform The integration runs inference through a single stream created with `create_inference_stream(network=...)`, which accepts a compiled model as a URL (downloaded into the model cache), a local file path, or an SDK model-zoo preset name. When no model is specified, the SDK model-zoo preset `yolox-s-coco-onnx` is used (no download); it can also be selected explicitly with `model.path: yolox-s-coco-onnx`. +Models must be compiled for the Metis architecture — see [Building a model](/frigate/installation#building-a-model) for the full walkthrough. + -:::info - -The Metis board needs models compiled for its architecture. Compile them from the Voyager model zoo with the SDK tooling, e.g. `axzoo build yolox_s_coco --backend axrelay --cal-source repr` (or compile a custom ONNX model with `axcompile`, see [Building a model](/frigate/installation#building-a-model)), producing a `.axm` file or a compiled model directory. Host the compiled model anywhere reachable, e.g. on Hugging Face, and point `model.path` at it. Plain strings in `model.path` are treated as SDK model-zoo preset names; anything else is treated as a file (downloaded from a URL or used in place). - -::: - ### Downloading YOLO Models #### Axelera Metis @@ -814,6 +810,4 @@ The Metis board needs models compiled for its architecture. Compile them from th The Metis board runs models compiled for its architecture (`.axm` files, or a compiled model directory). Two options are available: - **SDK model-zoo preset (no compilation or download needed):** the `yolox-s-coco-onnx` preset ships with the Voyager SDK (YOLOX, Megvii, Apache-2.0 licensed). Select it with `model.path: yolox-s-coco-onnx`. -- **Compiled model:** build one with the SDK tooling, e.g. `axzoo build yolox_s_coco --backend axrelay --cal-source repr` (model zoo) or `axcompile -i model.onnx -o compiled/` (custom ONNX). See [Building a model](/frigate/installation#building-a-model). - -Host the compiled model somewhere Frigate can reach (a local path or an HTTP URL, e.g. on Hugging Face) and point `model.path` at it. With no `model.path`, the detector uses the SDK model-zoo preset `yolox-s-coco-onnx` by default (no download). Set `model.width` and `model.height` to the compiled model's input size. +- **Compiled model:** see [Building a model](/frigate/installation#building-a-model) for how to compile one, then point `model.path` at it via a local path or an HTTP URL (e.g. on Hugging Face). Set `model.width` and `model.height` to the compiled model's input size. diff --git a/frigate/detectors/plugins/axelera.py b/frigate/detectors/plugins/axelera.py index ff7bbbe4a7..1d1153f9ed 100644 --- a/frigate/detectors/plugins/axelera.py +++ b/frigate/detectors/plugins/axelera.py @@ -24,12 +24,12 @@ Model configuration follows the hardware-backend convention: - `model.width` / `model.height`: input frame dimensions for the network. """ +import collections import logging import os import queue import threading import time -import urllib.request from typing import Literal import numpy as np @@ -38,6 +38,7 @@ from pydantic import ConfigDict from frigate.const import MODEL_CACHE_DIR from frigate.detectors.detection_api import DetectionApi from frigate.detectors.detector_config import BaseDetectorConfig +from frigate.util.downloader import ModelDownloader logger = logging.getLogger(__name__) @@ -48,10 +49,6 @@ DETECTOR_KEY = "axelera" # and cannot be referenced by Frigate (MIT), see blakeblackshear/frigate#10717. AX_DEFAULT_NETWORK = "yolox-s-coco-onnx" -# Optional download URL for a compiled default model. When empty the SDK -# model-zoo preset AX_DEFAULT_NETWORK is used instead (no download needed). -AX_DEFAULT_URL = "" - INFERENCE_TIMEOUT = 5.0 MAX_ROWS = 20 @@ -60,37 +57,48 @@ MAX_ROWS = 20 def is_url(value: str) -> bool: - return value.startswith(("http://", "https://", "www.")) + """Return True for well-formed http(s) URLs (scheme required).""" + return value.startswith(("http://", "https://")) + + +def _looks_like_path(value: str) -> bool: + """Heuristic: a value that should refer to a file on disk, not a preset.""" + return "/" in value or "." in value def _resolve_network(model_path, model_url) -> str: """Resolve the value passed to ``create_inference_stream(network=...)``. - Returns a download URL -> cached local file / existing local file / SDK - model-zoo preset name, mirroring the resolution hailo8l performs for ``.hef``. + Returns a download URL cached as a local file / an existing local file / + an SDK model-zoo preset name, mirroring the resolution hailo8l performs + for ``.hef``. """ if model_url and model_path: raise ValueError("`model.path` cannot be both a URL and a local path.") - if model_url or (not model_path and AX_DEFAULT_URL): - url = model_url or AX_DEFAULT_URL - name = os.path.basename(url) or AX_DEFAULT_NETWORK + if model_url: + name = os.path.basename(model_url) or AX_DEFAULT_NETWORK os.makedirs(MODEL_CACHE_DIR, exist_ok=True) destination = os.path.join(MODEL_CACHE_DIR, name) if not os.path.isfile(destination): - logger.info("axelera: downloading model from %s", url) + logger.info("axelera: downloading model from %s", model_url) try: - urllib.request.urlretrieve(url, destination) + ModelDownloader.download_from_url(model_url, destination) except Exception as exc: raise RuntimeError( - f"Failed to download model from {url}: {exc}" + f"Failed to download model from {model_url}: {exc}" ) from exc return destination if model_path: - if os.path.isfile(model_path): + if _looks_like_path(model_path): + if not os.path.isfile(model_path): + raise FileNotFoundError( + f"Model file not found: {model_path}. If you meant to use " + "an SDK model-zoo preset name, drop the file extension / path." + ) return model_path - # Not an existing file: treat as an SDK model-zoo preset name. + # Bare name: SDK model-zoo preset. return model_path return AX_DEFAULT_NETWORK @@ -120,13 +128,12 @@ class _AxeleraInference: never starves and is not torn down by its internal inference timeout during quiet periods (e.g. a camera with no motion). - Results are NOT strictly paired to request ids: per-frame pairing - desynchronises in production once keep-alive results interleave with real - frames (a request can receive a zero-frame result, or a real result gets - attributed to the wrong frame). Instead the pump keeps the most recent - completed result, and ``run()`` pushes its frame then waits for a result - newer than the sequence observed at call time and returns it. Frigate only - needs the latest per-camera detections; it correlates objects across frames. + Results are STRICTLY paired to requests: every frame the feeder pushes into + the SDK registers its request id on a deque (in submission order), and each + stream result pops the next pending id and stores the outcome under it. + ``run()`` waits for its own id only, so a result can never be attributed to + a different camera or crop, and primer/keep-alive results (unregistered ids) + are dropped instead of being returned to a caller. """ def __init__( @@ -136,8 +143,9 @@ class _AxeleraInference: self._width, self._height = int(width), int(height) self._frame_q: queue.Queue = queue.Queue(maxsize=64) self._cond = threading.Condition() - self._latest_seq = 0 - self._latest_result: np.ndarray | None = None + self._pending_ids: collections.deque = collections.deque() + self._results: dict[int, np.ndarray] = {} + self._next_id = 0 if stream_factory is None: # lazy import: never import the SDK unless the detector is used @@ -163,18 +171,26 @@ class _AxeleraInference: def _feed(self): # primer frames: the SDK init pre-buffers several frames before the - # source yields (validated on hardware). + # source yields (validated on hardware). Their ids are NOT registered, + # so their stream results are dropped by the pump. for _ in range(3): yield np.zeros((self._height, self._width, 3), np.uint8) while True: try: - frame = self._frame_q.get(timeout=1.0) + frame_id, frame = self._frame_q.get(timeout=1.0) except queue.Empty: # keep-alive: no request pending, keep the SDK stream fed so the - # inference timeout never kills it during quiet periods - frame = np.zeros((self._height, self._width, 3), np.uint8) + # inference timeout never kills it during quiet periods. The + # None id means this frame's result is dropped by the pump. + frame_id, frame = None, np.zeros( + (self._height, self._width, 3), np.uint8 + ) if frame is None: # shutdown sentinel from stop() return + if frame_id is not None: + # submission order == result order: the pump pops this id when + # the SDK yields the result for this frame. + self._pending_ids.append(frame_id) yield frame def _detection_rows(self, frame_result) -> list: @@ -184,7 +200,15 @@ class _AxeleraInference: them by the region size, pixel boxes are discarded as outside the frame (zero events despite working detections). """ - dets = getattr(frame_result, "detections", None) or [] + dets = getattr(frame_result, "detections", None) + if dets is None: + return [] + try: + # some SDK builds return a numpy array; `or []` on an array raises + # ValueError (ambiguous truth value) with 2+ detections + dets = list(dets) + except TypeError: + dets = [dets] rows = [] for det in dets: label = getattr(getattr(det, "label", None), "name", None) @@ -193,7 +217,6 @@ class _AxeleraInference: class_id = self._labelmap.get(str(label).strip()) if class_id is None: continue - score = 0.0 raw_box = getattr(det, "box", None) if raw_box is None: continue @@ -211,8 +234,8 @@ class _AxeleraInference: ( class_id, score, - y0 / self._height, - x0 / self._width, + max(0.0, y0) / self._height, + max(0.0, x0) / self._width, min(y1, self._height) / self._height, min(x1, self._width) / self._width, ) @@ -225,16 +248,20 @@ class _AxeleraInference: try: for frame_result in self._stream: out = _pad_rows(self._detection_rows(frame_result)) - with self._cond: - self._latest_seq += 1 - self._latest_result = out - self._cond.notify_all() + if self._pending_ids: + # FIFO: the stream returns results in submission order, so + # the oldest pending id belongs to this result. + frame_id = self._pending_ids.popleft() + with self._cond: + self._results[frame_id] = out + self._cond.notify_all() except Exception: logger.exception( "axelera: inference stream failed; detector requires restart" ) def run(self, tensor_input, timeout: float = INFERENCE_TIMEOUT) -> np.ndarray: + """Run one inference and return this frame's detections (see detect_raw).""" if tensor_input is None or getattr(tensor_input, "ndim", 0) < 3: raise ValueError( f"Unexpected frame shape: {getattr(tensor_input, 'shape', tensor_input)}" @@ -246,36 +273,45 @@ class _AxeleraInference: if frame.shape[0] != 1: raise ValueError(f"Unexpected batch dimension in frame: {frame.shape}") frame = frame[0] - frame = np.ascontiguousarray(frame) + # Real copy: the shared-memory camera buffer is unmapped by Frigate as + # soon as detect_raw returns, and our queue can hold the frame past that + # point (timeout path, backed-up queue). NP views would let the SDK read + # a torn/unmapped segment. + frame = np.array(frame, dtype=np.uint8, order="C", copy=True) + if not self._thread.is_alive(): # The pump is gone (stream wedged): never block on the queue put, # the SDK side is not draining it and the queue would fill up, # permanently wedging this detector. raise RuntimeError("Axelera inference thread stopped; restart required.") + + # Every submitted frame gets a unique id; the pump stores the result + # under this id, so we can wait for OUR frame's outcome exclusively. + self._next_id += 1 + frame_id = self._next_id try: - self._frame_q.put_nowait(frame) + self._frame_q.put((frame_id, frame), timeout=2.0) except queue.Full: - # The SDK side is not draining: drop the frame instead of blocking - # forever, then fall through to the latest-result wait below. - logger.debug("axelera: dropping frame, inference queue is full") + raise RuntimeError( + "Axelera inference queue is full; restart required." + ) from None deadline = time.monotonic() + timeout with self._cond: - start_seq = self._latest_seq - while self._latest_seq <= start_seq: + while frame_id not in self._results: remaining = deadline - time.monotonic() if remaining <= 0: break self._cond.wait(remaining) - if self._latest_seq > start_seq: - result = self._latest_result - if result is not None: - return result - if not self._thread.is_alive(): + if frame_id in self._results: + result = self._results.pop(frame_id) + return result + if self._thread.is_alive(): raise RuntimeError( - "Axelera inference thread stopped; restart required." - ) from None - return np.zeros((MAX_ROWS, 6), np.float32) + "Axelera inference stream returned no result for the submitted " + f"frame within {timeout:.1f}s (stream stall); restart required." + ) + raise RuntimeError("Axelera inference thread stopped; restart required.") def stop(self): # cleanup must not raise @@ -298,14 +334,13 @@ class AxeleraDetector(DetectionApi): type_key = DETECTOR_KEY def __init__(self, detector_config: "AxeleraDetectorConfig"): - super().__init__(detector_config) - model = detector_config.model if model is None: raise ValueError( f"type={DETECTOR_KEY}: the `model:` block is required " "(path to a compiled .axm, URL, or SDK model-zoo preset name)." ) + super().__init__(detector_config) self.width = model.width self.height = model.height @@ -337,9 +372,16 @@ class AxeleraDetector(DetectionApi): ) def detect_raw(self, tensor_input) -> np.ndarray: + """Run the detector on one RGB frame and return (20, 6) float32 rows. + + Each row is (class_id, score, ymin, xmin, ymax, xmax) in NORMALIZED + coordinates [0..1] relative to the model input size. Unused rows are + zero-filled. Detections are strictly paired to THIS frame's inference. + """ return self._inference.run(tensor_input) def close(self): + """Stop the inference stream and release the Metis device.""" self._inference.stop() def __del__(self): diff --git a/frigate/stats/util.py b/frigate/stats/util.py index af3a0b7892..f89118557a 100644 --- a/frigate/stats/util.py +++ b/frigate/stats/util.py @@ -344,7 +344,9 @@ async def set_npu_usages(config: FrigateConfig, all_stats: dict[str, Any]) -> No # Axelera Metis: temperature from the board controller. # There is no utilization readout without the axsystemserver # broker, which cannot run alongside a live detector. - metis_temp = get_axelera_board_temp() + # axcmd spawns can take seconds on a busy board; this stats task + # runs in the async event loop, so run the blocking read in a thread. + metis_temp = await asyncio.to_thread(get_axelera_board_temp) if metis_temp is not None: stats["axelera"] = {"mem": "-%", "temp": metis_temp} diff --git a/frigate/util/services.py b/frigate/util/services.py index 8542eb6f91..20c610dcf9 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -800,12 +800,27 @@ def get_axcl_npu_stats() -> dict[str, str | float] | None: return None +_board_temp_cache: tuple[float, float] | None = None +_BOARD_TEMP_TTL = 5.0 + + def get_axelera_board_temp() -> float | None: """Get the Axelera Metis board temperature through the SDK's axcmd tool. The read goes through the board controller and does not open an inference - stream, so it is safe to call while a detector is running. + stream, so it is safe to call while a detector is running. Results are + cached for a few seconds so the stats cycle (detector temperature + NPU + usage run in the same tick) spawns axcmd at most once. + + NOTE: this function is blocking (up to ~5s on a hung board). Callers in + async contexts must wrap it in ``asyncio.to_thread``. """ + global _board_temp_cache + + now = time.monotonic() + if _board_temp_cache is not None and now - _board_temp_cache[0] < _BOARD_TEMP_TTL: + return _board_temp_cache[1] + axcmd_path = None for candidate in ( @@ -826,6 +841,7 @@ def get_axelera_board_temp() -> float | None: capture_output=True, text=True, timeout=5, + check=False, ) except (OSError, sp.TimeoutExpired): return None @@ -833,16 +849,21 @@ def get_axelera_board_temp() -> float | None: if result.returncode != 0: return None - match = re.search(r"=\s*([\d.]+)", result.stdout) + # anchor on the board-controller line ("N: temp_sensorX@YY = ") so + # unrelated "= " output (firmware/version preambles) is not picked up + match = re.search(r"temp_sensor[^=\n]*=\s*([\d.]+)", result.stdout) if match is None: return None try: - return round(float(match.group(1)), 1) + temp = round(float(match.group(1)), 1) except ValueError: return None + _board_temp_cache = (now, temp) + return temp + def try_get_info(f, h, default="N/A", sensor=None): try: