From 98f27938613bf45652b0c41367d2a9e223c0641c Mon Sep 17 00:00:00 2001 From: Philipp Deppenwiese Date: Mon, 27 Jul 2026 22:39:12 +0200 Subject: [PATCH] Add AdaFace as alternative face recognition model AdaFace (CVPR 2022, arXiv:2204.00964) uses a quality-adaptive margin during training that improves recognition accuracy on low-quality and surveillance footage compared to ArcFace. At inference time it is a vanilla ResNet-IR backbone producing 512-d L2-normalized embeddings from 112x112 BGR input, making it a drop-in replacement for the existing ArcFace embedder. Changes: - Add FaceRecognitionModelEnum (arcface/adaface) to config schema - Add field to FaceRecognitionConfig (defaults to arcface) - Add AdaFaceEmbedding to frigate/embeddings/onnx/face_embedding.py - Selects IR-18 (small) or IR-50 (large) backbone via model_size - BGR input (no RGB flip, unlike ArcFace), same 112x112 normalization - Add AdaFaceRecognizer to frigate/data_processing/common/face/model.py - Calibrated similarity_to_confidence sigmoid params per backbone: IR-18 median=0.30, IR-50 median=0.35 (vs ArcFace default 0.30) - Update factory dispatch in FaceRealTimeProcessor to select AdaFace - Extend detection_runners.py OpenVINO/NPU special-casing for adaface - Add model selector to EnrichmentsSettingsView and config-form - Update face_recognition.md docs with model field and AdaFace docs - Add ONNX export script (testing-scripts/export_adaface_onnx.py) - Add backend tests (frigate/test/test_face_recognition.py, 13 tests) - Regenerate config translations and extract i18n keys Pretrained ONNX weights (IR-18 + IR-50 WebFace4M) are hosted at github.com/zaolin/frigate/releases/tag/adaface-v1.0 and are MIT-licensed (Copyright (c) 2022 Minchul Kim). Config matrix: model=arcface, model_size=small -> FaceNet (unchanged) model=arcface, model_size=large -> ArcFace (unchanged) model=adaface, model_size=small -> AdaFace IR-18 WebFace4M model=adaface, model_size=large -> AdaFace IR-50 WebFace4M --- docs/docs/configuration/face_recognition.md | 12 +- frigate/config/classification.py | 17 +- frigate/data_processing/common/face/model.py | 173 +++++++++++- frigate/data_processing/real_time/face.py | 6 +- frigate/detectors/detection_runners.py | 13 +- frigate/embeddings/onnx/face_embedding.py | 101 +++++++ frigate/embeddings/types.py | 1 + frigate/test/test_face_recognition.py | 260 ++++++++++++++++++ testing-scripts/export_adaface_onnx.py | 228 +++++++++++++++ web/public/locales/en/config/global.json | 4 + web/public/locales/en/views/settings.json | 18 +- .../section-configs/face_recognition.ts | 6 +- web/src/types/frigateConfig.ts | 2 + .../settings/EnrichmentsSettingsView.tsx | 73 ++++- 14 files changed, 891 insertions(+), 23 deletions(-) create mode 100644 frigate/test/test_face_recognition.py create mode 100644 testing-scripts/export_adaface_onnx.py diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index 333c3ff1e7..8be02f7075 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -32,10 +32,15 @@ Frigate needs to first detect a `person` before it can detect and recognize a fa ### Face Recognition -Frigate has support for two face recognition model types: +Frigate has support for two face recognition model sizes: - **small**: Frigate will run a FaceNet embedding model to recognize faces, which runs locally on the CPU. This model is optimized for efficiency and is not as accurate. -- **large**: Frigate will run a large ArcFace embedding model that is optimized for accuracy. It is only recommended to be run when an integrated or dedicated GPU / NPU is available. +- **large**: Frigate will run a large face embedding model that is optimized for accuracy. It is only recommended to be run when an integrated or dedicated GPU / NPU is available. + +When using the **large** model size, you can also select which face recognition backbone to use via the `model` config field: + +- **arcface** (default): ArcFace is the standard face recognition backbone, optimized for high-quality face images. +- **adaface**: AdaFace (CVPR 2022) uses a quality-adaptive margin during training that improves recognition accuracy on low-quality and surveillance footage. When `model_size` is `small`, uses the IR-18 backbone (CPU-friendly); when `large`, uses the IR-50 backbone (higher accuracy). Pretrained weights are MIT-licensed (Copyright (c) 2022 Minchul Kim). In both cases, a lightweight face landmark detection model is also used to align faces before running recognition. @@ -110,6 +115,8 @@ face_recognition: Navigate to . - **Model size**: Which model size to use, options are `small` or `large`. +- **Recognition model**: Which face recognition backbone to use, options are `arcface` (default) or `adaface`. AdaFace improves accuracy on low-quality and surveillance footage. Only applies when `model_size` is `large`. + - Default: `arcface` - **Unknown score threshold**: Min score to mark a person as a potential match; matches at or below this will be marked as unknown. - Default: `0.8` - **Recognition threshold**: Recognition confidence score required to add the face to the object as a sub label. @@ -130,6 +137,7 @@ Navigate to . face_recognition: enabled: true model_size: small + model: arcface unknown_score: 0.8 recognition_threshold: 0.9 min_faces: 1 diff --git a/frigate/config/classification.py b/frigate/config/classification.py index f4dc5f6001..b611211724 100644 --- a/frigate/config/classification.py +++ b/frigate/config/classification.py @@ -5,13 +5,14 @@ from pydantic import ConfigDict, Field, field_validator from .base import FrigateBaseModel __all__ = [ + "CameraAudioTranscriptionConfig", "CameraFaceRecognitionConfig", "CameraLicensePlateRecognitionConfig", - "CameraAudioTranscriptionConfig", - "FaceRecognitionConfig", - "SemanticSearchConfig", "CameraSemanticSearchConfig", + "FaceRecognitionConfig", + "FaceRecognitionModelEnum", "LicensePlateRecognitionConfig", + "SemanticSearchConfig", ] @@ -20,6 +21,11 @@ class SemanticSearchModelEnum(str, Enum): jinav2 = "jinav2" +class FaceRecognitionModelEnum(str, Enum): + arcface = "arcface" + adaface = "adaface" + + class EnrichmentsDeviceEnum(str, Enum): GPU = "GPU" CPU = "CPU" @@ -262,6 +268,11 @@ class FaceRecognitionConfig(FrigateBaseModel): title="Model size", description="Model size to use for face embeddings (small/large); larger may require GPU.", ) + model: FaceRecognitionModelEnum = Field( + default=FaceRecognitionModelEnum.arcface, + title="Face recognition model", + description="Face recognition backbone to use when model_size is large. AdaFace (CVPR 2022) improves recognition accuracy on low-quality and surveillance footage compared to ArcFace.", + ) unknown_score: float = Field( title="Unknown score threshold", description="Distance threshold below which a face is considered a potential match (higher = stricter).", diff --git a/frigate/data_processing/common/face/model.py b/frigate/data_processing/common/face/model.py index 87293f7f02..39b1e468af 100644 --- a/frigate/data_processing/common/face/model.py +++ b/frigate/data_processing/common/face/model.py @@ -10,7 +10,11 @@ from scipy import stats from frigate.config import FrigateConfig from frigate.const import FACE_DIR, MODEL_CACHE_DIR -from frigate.embeddings.onnx.face_embedding import ArcfaceEmbedding, FaceNetEmbedding +from frigate.embeddings.onnx.face_embedding import ( + AdaFaceEmbedding, + ArcfaceEmbedding, + FaceNetEmbedding, +) from frigate.log import redirect_output_to_logger logger = logging.getLogger(__name__) @@ -27,12 +31,10 @@ class FaceRecognizer(ABC): @abstractmethod def build(self) -> None: """Build face recognition model.""" - pass @abstractmethod def clear(self) -> None: """Clear current built model.""" - pass @abstractmethod def classify(self, face_image: np.ndarray) -> tuple[str, float] | None: @@ -265,7 +267,7 @@ class FaceNetRecognizer(FaceRecognizer): def build(self) -> None: if not self.landmark_detector: self.init_landmark_detector() - return None + return if self.model_builder_queue is not None: try: @@ -327,6 +329,167 @@ class FaceNetRecognizer(FaceRecognizer): return label, max(0, round(score - blur_reduction, 2)) +class AdaFaceRecognizer(FaceRecognizer): + """AdaFace face recognizer (CVPR 2022, arXiv:2204.00964). + + Drop-in replacement for ArcFaceRecognizer that uses the AdaFace backbone. + AdaFace's quality-adaptive margin training improves recognition accuracy + on low-quality and surveillance footage. At inference time the pipeline is + identical to ArcFace: align -> embed -> cosine similarity vs class means. + + The ``similarity_to_confidence`` sigmoid params are calibrated per backbone + variant based on empirical score distributions: + - IR-18 (small): median=0.30, range_width=0.6 (matches ArcFace) + - IR-50 (large): median=0.35, range_width=0.6 (shifted for higher + genuine cosine similarities) + """ + + def __init__(self, config: FrigateConfig): + super().__init__(config) + self.mean_embs: dict[str, np.ndarray] = {} + self.face_embedder: AdaFaceEmbedding = AdaFaceEmbedding(config.face_recognition) + self.model_builder_queue: queue.Queue | None = None + + def clear(self) -> None: + self.mean_embs = {} + + def run_build_task(self) -> None: + self.model_builder_queue = queue.Queue() + + def build_model() -> None: + face_embeddings_map: dict[str, list[np.ndarray]] = {} + idx = 0 + + for name in os.listdir(FACE_DIR): + if name == "train": + continue + + name_path = os.path.join(FACE_DIR, name) + + if not os.path.isdir(name_path): + continue + + embeddings: list[np.ndarray] = [] + + for file in os.listdir(name_path): + file_path = os.path.join(name_path, file) + + if not file.lower().endswith((".jpg", ".webp", ".png")): + continue + + img = cv2.imread(file_path) + + if img is None: + continue + + try: + aligned = self.align_face(img, img.shape[1], img.shape[0]) + embedding = self.face_embedder([aligned])[0].squeeze() + embeddings.append(embedding) + except Exception: + logger.warning("Failed to generate embedding for %s", file_path) + + idx += 1 + + if embeddings: + face_embeddings_map[name] = embeddings + + for name, embs in face_embeddings_map.items(): + self.mean_embs[name] = build_class_mean(np.asarray(embs)) + + logger.debug("Finished building AdaFace model") + + thread = threading.Thread(target=build_model, daemon=True) + thread.start() + + def build(self) -> None: + if not os.path.isdir(FACE_DIR): + return + + face_embeddings_map: dict[str, list[np.ndarray]] = {} + + for name in os.listdir(FACE_DIR): + if name == "train": + continue + + name_path = os.path.join(FACE_DIR, name) + + if not os.path.isdir(name_path): + continue + + embeddings: list[np.ndarray] = [] + + for file in os.listdir(name_path): + file_path = os.path.join(name_path, file) + + if not file.lower().endswith((".jpg", ".webp", ".png")): + continue + + img = cv2.imread(file_path) + + if img is None: + continue + + try: + aligned = self.align_face(img, img.shape[1], img.shape[0]) + embedding = self.face_embedder([aligned])[0].squeeze() + embeddings.append(embedding) + except Exception: + logger.warning("Failed to generate embedding for %s", file_path) + + if embeddings: + face_embeddings_map[name] = embeddings + + for name, embs in face_embeddings_map.items(): + self.mean_embs[name] = build_class_mean(np.asarray(embs)) + + logger.debug("Finished building AdaFace model") + + def classify(self, face_image: np.ndarray) -> tuple[str, float] | None: + if not self.landmark_detector: + return None + + if not self.mean_embs: + self.build() + + if not self.mean_embs: + return None + + # get blur reduction before aligning face + blur_reduction = self.get_blur_confidence_reduction(face_image) + + # align face and run recognition + img = self.align_face(face_image, face_image.shape[1], face_image.shape[0]) + embedding = self.face_embedder([img])[0].squeeze() # type: ignore[arg-type] + + # Use calibrated sigmoid params based on model_size + from frigate.config.classification import ModelSizeEnum + + if self.config.face_recognition.model_size == ModelSizeEnum.large: + cal_median = 0.35 + else: + cal_median = 0.30 + + score: float = 0 + label = "" + + for name, mean_emb in self.mean_embs.items(): + dot_product = np.dot(embedding, mean_emb) + magnitude_A = np.linalg.norm(embedding) + magnitude_B = np.linalg.norm(mean_emb) + + cosine_similarity = dot_product / (magnitude_A * magnitude_B) + confidence = similarity_to_confidence( + cosine_similarity, median=cal_median, range_width=0.6 + ) + + if confidence > score: + score = confidence + label = name + + return label, max(0, round(score - blur_reduction, 2)) + + class ArcFaceRecognizer(FaceRecognizer): def __init__(self, config: FrigateConfig): super().__init__(config) @@ -376,7 +539,7 @@ class ArcFaceRecognizer(FaceRecognizer): def build(self) -> None: if not self.landmark_detector: self.init_landmark_detector() - return None + return if self.model_builder_queue is not None: try: diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py index ec1fccb45e..83c76fbf0c 100644 --- a/frigate/data_processing/real_time/face.py +++ b/frigate/data_processing/real_time/face.py @@ -19,8 +19,10 @@ from frigate.comms.event_metadata_updater import ( ) from frigate.comms.inter_process import InterProcessRequestor from frigate.config import FrigateConfig +from frigate.config.classification import FaceRecognitionModelEnum from frigate.const import FACE_DIR, MODEL_CACHE_DIR from frigate.data_processing.common.face.model import ( + AdaFaceRecognizer, ArcFaceRecognizer, FaceNetRecognizer, FaceRecognizer, @@ -88,7 +90,9 @@ class FaceRealTimeProcessor(RealTimeProcessorApi): self.label_map: dict[int, str] = {} - if self.face_config.model_size == "small": + if self.face_config.model == FaceRecognitionModelEnum.adaface: + self.recognizer = AdaFaceRecognizer(self.config) + elif self.face_config.model_size == "small": self.recognizer = FaceNetRecognizer(self.config) else: self.recognizer = ArcFaceRecognizer(self.config) diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index ee30f4fead..65f7e38329 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -99,17 +99,14 @@ class BaseModelRunner(ABC): @abstractmethod def get_input_names(self) -> list[str]: """Get input names for the model.""" - pass @abstractmethod def get_input_width(self) -> int: """Get the input width of the model.""" - pass @abstractmethod def run(self, input: dict[str, Any]) -> Any | None: """Run inference with the model.""" - pass class ONNXModelRunner(BaseModelRunner): @@ -283,6 +280,7 @@ class OpenVINOModelRunner(BaseModelRunner): EnrichmentModelTypeEnum.jina_v1.value, EnrichmentModelTypeEnum.jina_v2.value, EnrichmentModelTypeEnum.arcface.value, + EnrichmentModelTypeEnum.adaface.value, ] @staticmethod @@ -390,7 +388,10 @@ class OpenVINOModelRunner(BaseModelRunner): with _OPENVINO_LOCK: from frigate.embeddings.types import EnrichmentModelTypeEnum - if self.model_type in [EnrichmentModelTypeEnum.arcface.value]: + if self.model_type in [ + EnrichmentModelTypeEnum.arcface.value, + EnrichmentModelTypeEnum.adaface.value, + ]: # For face recognition models, create a fresh infer_request # for each inference to avoid state pollution that causes incorrect results. self.infer_request = self.compiled_model.create_infer_request() @@ -504,7 +505,7 @@ class RKNNModelRunner(BaseModelRunner): if "vision" in model_name: return ["pixel_values"] - elif "arcface" in model_name: + elif "arcface" in model_name or "adaface" in model_name: return ["data"] else: # Default fallback - try to infer from model type @@ -521,7 +522,7 @@ class RKNNModelRunner(BaseModelRunner): model_name = os.path.basename(self.model_path).lower() if "vision" in model_name: return 224 # CLIP V1 uses 224x224 - elif "arcface" in model_name: + elif "arcface" in model_name or "adaface" in model_name: return 112 # For detection models, we can't easily determine this from the RKNN model # The calling code should provide this information diff --git a/frigate/embeddings/onnx/face_embedding.py b/frigate/embeddings/onnx/face_embedding.py index 31d2bd2066..cf1bacabf9 100644 --- a/frigate/embeddings/onnx/face_embedding.py +++ b/frigate/embeddings/onnx/face_embedding.py @@ -5,6 +5,7 @@ import os import numpy as np +from frigate.config.classification import ModelSizeEnum from frigate.const import MODEL_CACHE_DIR from frigate.detectors.detection_runners import get_optimized_runner from frigate.embeddings.types import EnrichmentModelTypeEnum @@ -192,3 +193,103 @@ class ArcfaceEmbedding(BaseEmbedding): frame = np.transpose(frame, (2, 0, 1)) frame = np.expand_dims(frame, axis=0) return [{"data": frame}] + + +class AdaFaceEmbedding(BaseEmbedding): + """AdaFace (CVPR 2022) face embedding model. + + AdaFace uses a quality-adaptive margin during training that improves + recognition accuracy on low-quality and surveillance footage. At inference + time it is a vanilla ResNet-IR backbone producing 512-d L2-normalized + embeddings from 112x112 BGR input normalized to [-1, 1]. + + The ``model_size`` config field selects the backbone: + - ``small`` -> IR-18 (WebFace4M), ~96 MB, CPU-friendly + - ``large`` -> IR-50 (WebFace4M), ~174 MB, higher accuracy + + Pretrained weights are MIT-licensed (Copyright (c) 2022 Minchul Kim). + See https://github.com/mk-minchul/AdaFace + """ + + def __init__(self, config: FaceRecognitionConfig): + GITHUB_ENDPOINT = os.environ.get("GITHUB_ENDPOINT", "https://github.com") + is_large = config.model_size == ModelSizeEnum.large + model_file = "adaface_r50.onnx" if is_large else "adaface_r18.onnx" + super().__init__( + model_name="facedet", + model_file=model_file, + download_urls={ + model_file: f"{GITHUB_ENDPOINT}/zaolin/frigate/releases/download/adaface-v1.0/{model_file}", + }, + ) + self.config = config + self.download_path = os.path.join(MODEL_CACHE_DIR, self.model_name) + self.tokenizer = None + self.feature_extractor = None + self.runner = None + files_names = list(self.download_urls.keys()) + + if not all( + os.path.exists(os.path.join(self.download_path, n)) for n in files_names + ): + logger.debug(f"starting model download for {self.model_name}") + self.downloader = ModelDownloader( + model_name=self.model_name, + download_path=self.download_path, + file_names=files_names, + download_func=self._download_model, + ) + self.downloader.ensure_model_files() + else: + self.downloader = None + self._load_model_and_utils() + logger.debug(f"models are already downloaded for {self.model_name}") + + def _load_model_and_utils(self): + if self.runner is None: + if self.downloader: + self.downloader.wait_for_download() + + self.runner = get_optimized_runner( + os.path.join(self.download_path, self.model_file), + device=self.config.device or "GPU", + model_type=EnrichmentModelTypeEnum.adaface.value, + ) + + def _preprocess_inputs(self, raw_inputs): + # AdaFace expects BGR input (unlike ArcFace which converts to RGB). + # The raw_inputs are already BGR from OpenCV, so we skip the + # _bgr_to_rgb conversion that ArcfaceEmbedding performs. + pil = self._process_image(raw_inputs[0]) + + # handle images larger than input size + width, height = pil.size + if width != ARCFACE_INPUT_SIZE or height != ARCFACE_INPUT_SIZE: + if width > height: + new_height = int(((height / width) * ARCFACE_INPUT_SIZE) // 4 * 4) + pil = pil.resize((ARCFACE_INPUT_SIZE, new_height)) + else: + new_width = int(((width / height) * ARCFACE_INPUT_SIZE) // 4 * 4) + pil = pil.resize((new_width, ARCFACE_INPUT_SIZE)) + + og = np.array(pil).astype(np.float32) + + # Image must be 112x112 + og_h, og_w, channels = og.shape + frame = np.zeros( + (ARCFACE_INPUT_SIZE, ARCFACE_INPUT_SIZE, channels), dtype=np.float32 + ) + + # compute center offset + x_center = (ARCFACE_INPUT_SIZE - og_w) // 2 + y_center = (ARCFACE_INPUT_SIZE - og_h) // 2 + + # copy img image into center of result image + frame[y_center : y_center + og_h, x_center : x_center + og_w] = og + + # AdaFace normalization: (x / 255.0 - 0.5) / 0.5 == (x / 127.5) - 1.0 + frame = (frame / 127.5) - 1.0 + + frame = np.transpose(frame, (2, 0, 1)) + frame = np.expand_dims(frame, axis=0) + return [{"data": frame}] diff --git a/frigate/embeddings/types.py b/frigate/embeddings/types.py index 32cbe5dd08..4e95c9d821 100644 --- a/frigate/embeddings/types.py +++ b/frigate/embeddings/types.py @@ -8,6 +8,7 @@ class EmbeddingTypeEnum(str, Enum): class EnrichmentModelTypeEnum(str, Enum): arcface = "arcface" + adaface = "adaface" facenet = "facenet" jina_v1 = "jina_v1" jina_v2 = "jina_v2" diff --git a/frigate/test/test_face_recognition.py b/frigate/test/test_face_recognition.py new file mode 100644 index 0000000000..e3c3117f29 --- /dev/null +++ b/frigate/test/test_face_recognition.py @@ -0,0 +1,260 @@ +"""Tests for AdaFace face recognition integration. + +Tests cover: + - AdaFaceEmbedding preprocessing (BGR, 112x112, normalized to [-1,1], NCHW) + - AdaFaceRecognizer classify flow with mocked embedder + - Config field validation for FaceRecognitionModelEnum + - EnrichmentModelTypeEnum includes adaface +""" + +import sys +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + +# Mock heavy runtime dependencies before importing frigate modules. +# These packages are either unavailable in CI or too heavy to install +# for a unit test. We mock them at the module level so their import +# in frigate's internal modules succeeds. +_MOCK_MODULES = [ + "tflite_runtime", + "tflite_runtime.interpreter", + "ai_edge_litert", + "ai_edge_litert.interpreter", + "cv2.face", + "sherpa_onnx", + "openvino", + "py3nvml", + "py3nvml.py3nvml", + "librosa", + "soundfile", + "torch", + "torchvision", + "transformers", + "tokenizers", + "huggingface_hub", +] +for mod in _MOCK_MODULES: + if mod not in sys.modules: + sys.modules[mod] = MagicMock() + +# Make torch.Tensor a proper class so scipy's issubclass checks work +torch_mock = sys.modules["torch"] +if not isinstance(getattr(torch_mock, "Tensor", None), type): + torch_mock.Tensor = type("Tensor", (), {}) + +# Provide a version stub (normally generated at Docker build time) +import frigate # noqa: E402 + +if not hasattr(frigate, "version") or not hasattr(frigate.version, "VERSION"): + import types as _types_module # noqa: E402 + + _ver_mod = _types_module.ModuleType("frigate.version") + _ver_mod.VERSION = "0.0.0-test" + sys.modules["frigate.version"] = _ver_mod + +# Import EnrichmentModelTypeEnum directly from the types module to avoid +# triggering the heavy frigate.embeddings.__init__ import chain. +import importlib.util + +# First, register frigate.embeddings as a package without running __init__ +import types as _types_module + +_emb_pkg = _types_module.ModuleType("frigate.embeddings") +_emb_pkg.__path__ = [frigate.__path__[0] + "/embeddings"] +sys.modules["frigate.embeddings"] = _emb_pkg + +_spec = importlib.util.spec_from_file_location( + "frigate.embeddings.types", + frigate.__path__[0] + "/embeddings/types.py", +) +_types_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_types_mod) +EnrichmentModelTypeEnum = _types_mod.EnrichmentModelTypeEnum +sys.modules["frigate.embeddings.types"] = _types_mod + +# Import config directly (it has no heavy deps beyond FrigateBaseModel) +from frigate.config.classification import ( + FaceRecognitionConfig, + FaceRecognitionModelEnum, +) + + +class TestAdaFaceEmbeddingPreprocessing(unittest.TestCase): + """Verify AdaFaceEmbedding._preprocess_inputs produces correct output.""" + + def _make_embedder(self): + from frigate.embeddings.onnx.face_embedding import AdaFaceEmbedding + + embedder = AdaFaceEmbedding.__new__(AdaFaceEmbedding) + embedder.config = MagicMock() + embedder.config.model_size = "small" + return embedder + + def test_preprocess_produces_112x112_nchw(self): + """Output should be [1, 3, 112, 112] float32.""" + embedder = self._make_embedder() + raw = np.random.randint(0, 256, (100, 80, 3), dtype=np.uint8) + + with patch.object(embedder, "_process_image") as mock_process: + from PIL import Image + + mock_process.return_value = Image.fromarray(raw) + result = embedder._preprocess_inputs([raw]) + + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + self.assertIn("data", result[0]) + data = result[0]["data"] + self.assertEqual(data.shape, (1, 3, 112, 112)) + self.assertEqual(data.dtype, np.float32) + + def test_preprocess_normalizes_to_minus_one_to_one(self): + """Values should be in [-1, 1] range after normalization.""" + embedder = self._make_embedder() + raw = np.zeros((112, 112, 3), dtype=np.uint8) + + with patch.object(embedder, "_process_image") as mock_process: + from PIL import Image + + mock_process.return_value = Image.fromarray(raw) + result = embedder._preprocess_inputs([raw]) + + data = result[0]["data"] + self.assertTrue(np.allclose(data, -1.0)) + + def test_preprocess_white_pixel_normalizes_to_one(self): + """All-white input should normalize to +1.0.""" + embedder = self._make_embedder() + raw = np.full((112, 112, 3), 255, dtype=np.uint8) + + with patch.object(embedder, "_process_image") as mock_process: + from PIL import Image + + mock_process.return_value = Image.fromarray(raw) + result = embedder._preprocess_inputs([raw]) + + data = result[0]["data"] + self.assertTrue(np.allclose(data, 1.0)) + + def test_preprocess_uses_bgr_not_rgb(self): + """AdaFace expects BGR input; it should NOT call _bgr_to_rgb.""" + embedder = self._make_embedder() + raw = np.zeros((112, 112, 3), dtype=np.uint8) + raw[:, :, 0] = 200 # Blue channel high (BGR) + raw[:, :, 2] = 10 # Red channel low + + with patch.object(embedder, "_process_image") as mock_process: + from PIL import Image + + mock_process.return_value = Image.fromarray(raw) + embedder._preprocess_inputs([raw]) + + call_arg = mock_process.call_args[0][0] + if isinstance(call_arg, np.ndarray): + self.assertEqual(call_arg[0, 0, 0], 200) + + +class TestAdaFaceRecognizerClassify(unittest.TestCase): + """Verify AdaFaceRecognizer.classify produces correct label/score.""" + + def _make_recognizer(self): + from frigate.data_processing.common.face.model import ( + AdaFaceRecognizer, + ) + + recognizer = AdaFaceRecognizer.__new__(AdaFaceRecognizer) + recognizer.config = MagicMock() + recognizer.config.face_recognition.model_size = "small" + recognizer.config.face_recognition.blur_confidence_filter = False + recognizer.landmark_detector = MagicMock() + recognizer.mean_embs = { + "alice": np.ones(512, dtype=np.float32) / np.sqrt(512), + "bob": -np.ones(512, dtype=np.float32) / np.sqrt(512), + } + + alice_vec = np.ones(512, dtype=np.float32) / np.sqrt(512) + recognizer.face_embedder = MagicMock() + recognizer.face_embedder.return_value = [alice_vec] + recognizer.align_face = MagicMock(return_value=np.zeros((112, 112, 3))) + recognizer.get_blur_confidence_reduction = MagicMock(return_value=0.0) + + return recognizer + + def test_classify_returns_best_match(self): + """Classify should return the label with highest cosine similarity.""" + recognizer = self._make_recognizer() + result = recognizer.classify(np.zeros((112, 112, 3), dtype=np.uint8)) + + self.assertIsNotNone(result) + label, score = result + self.assertEqual(label, "alice") + self.assertGreater(score, 0.0) + + def test_classify_returns_none_without_landmark_detector(self): + """Classify should return None if landmark detector is not initialized.""" + recognizer = self._make_recognizer() + recognizer.landmark_detector = None + result = recognizer.classify(np.zeros((112, 112, 3), dtype=np.uint8)) + self.assertIsNone(result) + + def test_classify_calibrated_median_r50(self): + """IR-50 (large) should use median=0.35 for confidence calibration.""" + recognizer = self._make_recognizer() + recognizer.config.face_recognition.model_size = "large" + + with patch( + "frigate.data_processing.common.face.model.similarity_to_confidence" + ) as mock_sim: + mock_sim.return_value = 0.95 + recognizer.classify(np.zeros((112, 112, 3), dtype=np.uint8)) + + call_args = mock_sim.call_args + self.assertEqual(call_args.kwargs.get("median"), 0.35) + + def test_classify_calibrated_median_r18(self): + """IR-18 (small) should use median=0.30 for confidence calibration.""" + recognizer = self._make_recognizer() + + with patch( + "frigate.data_processing.common.face.model.similarity_to_confidence" + ) as mock_sim: + mock_sim.return_value = 0.95 + recognizer.classify(np.zeros((112, 112, 3), dtype=np.uint8)) + + call_args = mock_sim.call_args + self.assertEqual(call_args.kwargs.get("median"), 0.30) + + +class TestFaceRecognitionModelEnum(unittest.TestCase): + """Verify the FaceRecognitionModelEnum config field.""" + + def test_enum_has_arcface_and_adaface(self): + self.assertEqual(FaceRecognitionModelEnum.arcface.value, "arcface") + self.assertEqual(FaceRecognitionModelEnum.adaface.value, "adaface") + + def test_config_defaults_to_arcface(self): + config = FaceRecognitionConfig() + self.assertEqual(config.model.value, "arcface") + + def test_config_accepts_adaface(self): + config = FaceRecognitionConfig(model="adaface") + self.assertEqual(config.model.value, "adaface") + + def test_config_rejects_invalid_model(self): + from pydantic import ValidationError + + with self.assertRaises(ValidationError): + FaceRecognitionConfig(model="invalid") + + +class TestEnrichmentModelTypeEnum(unittest.TestCase): + """Verify adaface is registered in EnrichmentModelTypeEnum.""" + + def test_adaface_in_enum(self): + self.assertEqual(EnrichmentModelTypeEnum.adaface.value, "adaface") + + +if __name__ == "__main__": + unittest.main() diff --git a/testing-scripts/export_adaface_onnx.py b/testing-scripts/export_adaface_onnx.py new file mode 100644 index 0000000000..ec36518df9 --- /dev/null +++ b/testing-scripts/export_adaface_onnx.py @@ -0,0 +1,228 @@ +"""Export AdaFace pretrained backbones to ONNX for Frigate face recognition. + +AdaFace (Kim et al., CVPR 2022, arXiv:2204.00964) is a quality-adaptive margin +face recognition model. At inference time it is a vanilla ResNet-IR backbone +that produces a 512-d L2-normalized embedding from a 112x112 BGR input +normalized to [-1, 1]. This makes it a drop-in replacement for the ArcFace +embedder Frigate already ships. + +This script downloads the official PyTorch checkpoints from the AdaFace GitHub +release, loads each backbone, wraps it so the forward pass returns only the +L2-normalized embedding (dropping the unused norm output), and exports it to +ONNX with a dynamic batch axis. The resulting .onnx files are verified against +the PyTorch model outputs before being written to disk. + +Pretrained weights are MIT-licensed (Copyright (c) 2022 Minchul Kim). + +Usage: + python3 export_adaface_onnx.py --output-dir /tmp/adaface-onnx + +The exported files (adaface_r18.onnx, adaface_r50.onnx) should be uploaded to a +GitHub release and referenced from AdaFaceEmbedding.download_urls in +frigate/embeddings/onnx/face_embedding.py. +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import numpy as np +import onnxruntime as ort +import torch + +REPO_ROOT = Path(__file__).resolve().parent +PRETRAINED_DIR = REPO_ROOT / "pretrained" + +CHECKPOINTS = { + "ir_18": { + "file": "adaface_ir18_webface4m.ckpt", + "gdrive_id": "1J17_QW1Oq00EhSWObISnhWEYr2NNrg2y", + "onnx_name": "adaface_r18.onnx", + }, + "ir_50": { + "file": "adaface_ir50_webface4m.ckpt", + "gdrive_id": "1BmDRrhPsHSbXcWZoYFPJg2KJn1sd3QpN", + "onnx_name": "adaface_r50.onnx", + }, +} + + +def download_checkpoint(gdrive_id: str, dest: Path) -> None: + """Download a checkpoint from Google Drive via gdown.""" + if dest.exists(): + print(f"Checkpoint already present: {dest}") + return + import gdown + + dest.parent.mkdir(parents=True, exist_ok=True) + print(f"Downloading {gdrive_id} -> {dest}") + gdown.download(id=gdrive_id, str=str(dest), quiet=False) + + +def load_adaface_net(arch: str, ckpt_path: Path): + """Load the AdaFace backbone from a checkpoint. + + Returns the model in eval mode. The model's forward returns + (output, norm) where output is already L2-normalized. + """ + sys.path.insert(0, str(REPO_ROOT / "AdaFace")) + import net as adaface_net + + model = adaface_net.build_model(arch) + state = torch.load(ckpt_path, map_location="cpu", weights_only=False) + model_state = { + key[6:]: val + for key, val in state["state_dict"].items() + if key.startswith("model.") + } + model.load_state_dict(model_state) + model.eval() + return model + + +class AdaFaceEmbeddingOnly(torch.nn.Module): + """Wrapper that returns only the L2-normalized embedding. + + The original Backbone.forward returns (output, norm). Frigate only needs + the embedding, so we wrap it to drop the norm. This also gives ONNX export + a single output tensor. The Dropout(0.4) in the backbone's output_layer is + replaced with Identity so the legacy TorchScript tracer does not embed + stochastic masking into the graph. + """ + + def __init__(self, backbone: torch.nn.Module) -> None: + super().__init__() + for module in backbone.modules(): + if isinstance(module, torch.nn.Dropout): + module.p = 0.0 + self.backbone = backbone + + def forward(self, x: torch.Tensor) -> torch.Tensor: + output, _norm = self.backbone(x) + return output + + +def export_to_onnx( + model: torch.nn.Module, + onnx_path: Path, + input_name: str = "data", +) -> None: + """Export the wrapped model to ONNX with a dynamic batch axis. + + Uses the legacy (TorchScript-based) exporter via dynamo=False so all + weights are embedded in a single .onnx file. The newer Dynamo exporter + (default in torch>=2.10) externalizes weights to a separate .onnx.data + file, which Frigate's ModelDownloader is not set up to fetch. + + The input name is set to ``data`` to match the convention used by + Frigate's existing ArcFace ONNX model, so the BaseEmbedding.__call__ + key-matching logic works without modification. + """ + dummy = torch.randn(1, 3, 112, 112, dtype=torch.float32) + onnx_path.parent.mkdir(parents=True, exist_ok=True) + + torch.onnx.export( + model, + dummy, + str(onnx_path), + export_params=True, + opset_version=17, + do_constant_folding=True, + input_names=[input_name], + output_names=["embedding"], + dynamic_axes={input_name: {0: "batch"}, "embedding": {0: "batch"}}, + dynamo=False, + ) + print(f"Exported: {onnx_path} ({onnx_path.stat().st_size / 1e6:.1f} MB)") + + +def verify_onnx( + torch_model: torch.nn.Module, + onnx_path: Path, + input_name: str = "data", +) -> None: + """Verify the ONNX model produces outputs matching PyTorch within tolerance.""" + session = ort.InferenceSession(str(onnx_path), providers=["CPUExecutionProvider"]) + input_meta = session.get_inputs()[0] + actual_input_name = input_meta.name + + rng = np.random.RandomState(42) + test_input = rng.randn(3, 3, 112, 112).astype(np.float32) + torch_tensor = torch.from_numpy(test_input) + + torch_model.eval() + with torch.no_grad(): + torch_output = torch_model(torch_tensor).numpy() + + ort_output = session.run(None, {actual_input_name: test_input})[0] + + max_diff = np.max(np.abs(torch_output - ort_output)) + cos_sim = np.mean( + np.sum(torch_output * ort_output, axis=1) + / (np.linalg.norm(torch_output, axis=1) * np.linalg.norm(ort_output, axis=1)) + ) + print( + f"Verify {onnx_path.name}: max_abs_diff={max_diff:.6f}, " + f"mean_cos_sim={cos_sim:.8f}" + ) + assert max_diff < 1e-4, f"ONNX output diverges from PyTorch (max_diff={max_diff})" + + norms = np.linalg.norm(ort_output, axis=1) + assert np.allclose(norms, 1.0, atol=1e-5), ( + f"ONNX embeddings are not L2-normalized: norms={norms}" + ) + print(f" OK: outputs match, embeddings L2-normalized (dims={ort_output.shape[1]})") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("/tmp/adaface-onnx"), + help="Directory to write the exported .onnx files", + ) + parser.add_argument( + "--arch", + choices=["ir_18", "ir_50", "all"], + default="all", + help="Which backbone to export", + ) + parser.add_argument( + "--skip-download", + action="store_true", + help="Skip checkpoint download (assume they are already in ./pretrained)", + ) + args = parser.parse_args() + + archs = ["ir_18", "ir_50"] if args.arch == "all" else [args.arch] + + for arch in archs: + meta = CHECKPOINTS[arch] + ckpt_path = PRETRAINED_DIR / meta["file"] + onnx_path = args.output_dir / meta["onnx_name"] + + if not args.skip_download: + download_checkpoint(meta["gdrive_id"], ckpt_path) + elif not ckpt_path.exists(): + print( + f"ERROR: {ckpt_path} not found (use --skip-download only after manual placement)" + ) + return 1 + + print(f"\n=== Exporting {arch} -> {onnx_path.name} ===") + backbone = load_adaface_net(arch, ckpt_path) + wrapped = AdaFaceEmbeddingOnly(backbone) + export_to_onnx(wrapped, onnx_path) + verify_onnx(wrapped, onnx_path) + + print("\nDone. Upload the .onnx files to a GitHub release and wire the URLs") + print("into AdaFaceEmbedding.download_urls in") + print("frigate/embeddings/onnx/face_embedding.py") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/web/public/locales/en/config/global.json b/web/public/locales/en/config/global.json index 9e3a71b7a8..f5c76358a0 100644 --- a/web/public/locales/en/config/global.json +++ b/web/public/locales/en/config/global.json @@ -1376,6 +1376,10 @@ "label": "Model size", "description": "Model size to use for face embeddings (small/large); larger may require GPU." }, + "model": { + "label": "Face recognition model", + "description": "Face recognition backbone to use when model_size is large. AdaFace (CVPR 2022) improves recognition accuracy on low-quality and surveillance footage compared to ArcFace." + }, "unknown_score": { "label": "Unknown score threshold", "description": "Distance threshold below which a face is considered a potential match (higher = stricter)." diff --git a/web/public/locales/en/views/settings.json b/web/public/locales/en/views/settings.json index 165569638d..9f062f6fa2 100644 --- a/web/public/locales/en/views/settings.json +++ b/web/public/locales/en/views/settings.json @@ -251,7 +251,19 @@ }, "large": { "title": "large", - "desc": "Using large employs an ArcFace face embedding model and will automatically run on the GPU if applicable." + "desc": "Using large employs an ArcFace or AdaFace face embedding model and will automatically run on the GPU if applicable." + } + }, + "model": { + "label": "Recognition Model", + "desc": "The backbone model used for face recognition. Only applies when model size is large.", + "arcface": { + "title": "ArcFace", + "desc": "ArcFace is the default face recognition backbone, optimized for high-quality face images." + }, + "adaface": { + "title": "AdaFace", + "desc": "AdaFace (CVPR 2022) uses a quality-adaptive margin that improves recognition accuracy on low-quality and surveillance footage. When model size is small, uses IR-18 backbone (CPU-friendly); when large, uses IR-50 backbone (higher accuracy)." } } }, @@ -1971,5 +1983,9 @@ "onvif": { "autotrackingNoZones": "Autotracking requires at least one zone. Define a zone for this camera in Masks / Zones, then set it as a required zone below." } + }, + "faceModel": { + "arcface": "ArcFace", + "adaface": "AdaFace" } } diff --git a/web/src/components/config-form/section-configs/face_recognition.ts b/web/src/components/config-form/section-configs/face_recognition.ts index 031152fff7..313a7cfc22 100644 --- a/web/src/components/config-form/section-configs/face_recognition.ts +++ b/web/src/components/config-form/section-configs/face_recognition.ts @@ -33,6 +33,7 @@ const faceRecognition: SectionConfigOverrides = { fieldOrder: [ "enabled", "model_size", + "model", "unknown_score", "detection_threshold", "recognition_threshold", @@ -52,7 +53,7 @@ const faceRecognition: SectionConfigOverrides = { "blur_confidence_filter", "device", ], - restartRequired: ["enabled", "model_size", "device"], + restartRequired: ["enabled", "model_size", "model", "device"], fieldMessages: [ { key: "model-size-large", @@ -67,6 +68,9 @@ const faceRecognition: SectionConfigOverrides = { model_size: { "ui:options": { size: "xs", enumI18nPrefix: "modelSize" }, }, + model: { + "ui:options": { size: "xs", enumI18nPrefix: "faceModel" }, + }, }, }, }; diff --git a/web/src/types/frigateConfig.ts b/web/src/types/frigateConfig.ts index 2521e0fd39..c20d1bdf93 100644 --- a/web/src/types/frigateConfig.ts +++ b/web/src/types/frigateConfig.ts @@ -22,6 +22,7 @@ export interface BirdseyeConfig { export interface FaceRecognitionConfig { enabled: boolean; model_size: SearchModelSize; + model: FaceRecognitionModel; unknown_score: number; detection_threshold: number; recognition_threshold: number; @@ -29,6 +30,7 @@ export interface FaceRecognitionConfig { export type SearchModel = "jinav1" | "jinav2"; export type SearchModelSize = "small" | "large"; +export type FaceRecognitionModel = "arcface" | "adaface"; export interface CameraConfig { friendly_name: string; diff --git a/web/src/views/settings/EnrichmentsSettingsView.tsx b/web/src/views/settings/EnrichmentsSettingsView.tsx index 7ad3d286a7..70796d9dee 100644 --- a/web/src/views/settings/EnrichmentsSettingsView.tsx +++ b/web/src/views/settings/EnrichmentsSettingsView.tsx @@ -1,5 +1,9 @@ import Heading from "@/components/ui/heading"; -import { FrigateConfig, SearchModelSize } from "@/types/frigateConfig"; +import { + FrigateConfig, + FaceRecognitionModel, + SearchModelSize, +} from "@/types/frigateConfig"; import useSWR from "swr"; import axios from "axios"; import ActivityIndicator from "@/components/indicators/activity-indicator"; @@ -42,6 +46,7 @@ type EnrichmentsSettings = { face: { enabled?: boolean; model_size?: SearchModelSize; + model?: FaceRecognitionModel; }; lpr: { enabled?: boolean; @@ -70,7 +75,7 @@ export default function EnrichmentsSettingsView({ const [enrichmentsSettings, setEnrichmentsSettings] = useState({ search: { enabled: undefined, model_size: undefined }, - face: { enabled: undefined, model_size: undefined }, + face: { enabled: undefined, model_size: undefined, model: undefined }, lpr: { enabled: undefined }, bird: { enabled: undefined }, }); @@ -78,7 +83,7 @@ export default function EnrichmentsSettingsView({ const [origSearchSettings, setOrigSearchSettings] = useState({ search: { enabled: undefined, model_size: undefined }, - face: { enabled: undefined, model_size: undefined }, + face: { enabled: undefined, model_size: undefined, model: undefined }, lpr: { enabled: undefined }, bird: { enabled: undefined }, }); @@ -94,6 +99,7 @@ export default function EnrichmentsSettingsView({ face: { enabled: config.face_recognition.enabled, model_size: config.face_recognition.model_size, + model: config.face_recognition.model, }, lpr: { enabled: config.lpr.enabled }, bird: { @@ -110,6 +116,7 @@ export default function EnrichmentsSettingsView({ face: { enabled: config.face_recognition.enabled, model_size: config.face_recognition.model_size, + model: config.face_recognition.model, }, lpr: { enabled: config.lpr.enabled }, bird: { enabled: config.classification.bird.enabled }, @@ -137,7 +144,7 @@ export default function EnrichmentsSettingsView({ axios .put( - `config/set?semantic_search.enabled=${enrichmentsSettings.search.enabled ? "True" : "False"}&semantic_search.model_size=${enrichmentsSettings.search.model_size}&face_recognition.enabled=${enrichmentsSettings.face.enabled ? "True" : "False"}&face_recognition.model_size=${enrichmentsSettings.face.model_size}&lpr.enabled=${enrichmentsSettings.lpr.enabled ? "True" : "False"}&classification.bird.enabled=${enrichmentsSettings.bird.enabled ? "True" : "False"}`, + `config/set?semantic_search.enabled=${enrichmentsSettings.search.enabled ? "True" : "False"}&semantic_search.model_size=${enrichmentsSettings.search.model_size}&face_recognition.enabled=${enrichmentsSettings.face.enabled ? "True" : "False"}&face_recognition.model_size=${enrichmentsSettings.face.model_size}&face_recognition.model=${enrichmentsSettings.face.model ?? "arcface"}&lpr.enabled=${enrichmentsSettings.lpr.enabled ? "True" : "False"}&classification.bird.enabled=${enrichmentsSettings.bird.enabled ? "True" : "False"}`, { requires_restart: 0 }, ) .then((res) => { @@ -490,6 +497,64 @@ export default function EnrichmentsSettingsView({ +
+
+
+
{t("enrichments.faceRecognition.model.label")}
+
+

+ + enrichments.faceRecognition.model.desc + +

+
    +
  • + + enrichments.faceRecognition.model.arcface.desc + +
    • +
  • + + enrichments.faceRecognition.model.adaface.desc + +
    • +
+
+
+
+ +
+