diff --git a/.env.example b/.env.example index a3d0872..eb99246 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,5 @@ AI_PRIVACY_ENABLE_IDENTITY_EXCLUSION=true AI_PRIVACY_IDENTITY_RECHECK_INTERVAL=12 AI_PRIVACY_IDENTITY_NEGATIVE_RECHECK_INTERVAL=90 AI_PRIVACY_ME_IMAGE_PATH=me.jpeg -AI_PRIVACY_FACE_MODEL_PATH=models/yolov12n-face.pt -AI_PRIVACY_FACE_DETECTION_MODEL_PATH=models/face_detection_yunet_2023mar.onnx +AI_PRIVACY_FACE_MODEL_PATH=models/yolo11s-pose_widerface.pt AI_PRIVACY_FACE_RECOGNITION_MODEL_PATH=models/face_recognition_sface_2021dec.onnx diff --git a/README.md b/README.md index 432117b..0446049 100644 --- a/README.md +++ b/README.md @@ -24,33 +24,70 @@ Mac 로컬 실행은 위와 같이 CPU 모드를 사용한다. 운영 환경은 ## AI privacy runtime -`privacy_blur`는 모든 WebRTC 스트림이 YOLOv12n-Face, YuNet, SFace 모델 세트를 -공유하고 각 스트림은 독립적인 얼굴 추적 상태만 유지한다. 공유 워커가 짧은 시간 -동안 들어온 프레임을 최대 `AI_PRIVACY_MAX_BATCH_SIZE`개까지 모아 한 번에 추론하며, -대기열은 `AI_PRIVACY_MAX_PENDING_FRAMES`로 제한된다. +`privacy_blur`는 모든 WebRTC 스트림이 하나의 GPU 런타임과 YOLO11s face pose, +SFace 모델 세트를 공유한다. 각 스트림은 독립적인 얼굴 추적 상태와 `client_id`별 +참조 얼굴 설정만 따로 가진다. 공유 프레임 워커가 짧은 시간 동안 들어온 프레임을 +최대 `AI_PRIVACY_MAX_BATCH_SIZE`개까지 모아 한 번에 추론하며, 대기열은 +`AI_PRIVACY_MAX_PENDING_FRAMES`로 제한된다. 처리 순서는 다음과 같다. -1. YOLO FP16 배치로 얼굴을 탐지한다. -2. YOLO가 함께 출력한 5개 랜드마크로 얼굴을 정렬하고 SFace 배치로 식별한다. - 랜드마크가 유효하지 않은 얼굴만 YuNet을 폴백으로 사용한다. -3. 참조 얼굴(`AI_PRIVACY_ME_IMAGE_PATH` 또는 `/reference-face` API)을 제외한 - 얼굴에 확장 타원 마스크와 Gaussian blur를 GPU에서 적용한다. -4. 처리 결과를 원래 WebRTC 트랙으로 반환한다. +1. YOLO FP16 배치로 얼굴과 5개 랜드마크를 탐지한다. +2. 얼굴 추적 상태를 갱신하고, 식별 재확인이 필요한 트랙만 비동기 식별 큐에 넣는다. + 랜드마크가 유효하지 않은 얼굴은 본인 제외 후보로 사용하지 않는다. +3. 현재까지 확정된 본인 트랙을 제외한 얼굴에 확장 타원 마스크와 Gaussian blur를 + GPU에서 적용한다. 프레임 처리는 SFace 식별 결과를 기다리지 않는다. +4. 별도 식별 워커가 YOLO 랜드마크로 얼굴을 정렬하고 SFace 배치로 참조 얼굴과 + 비교한다. 결과는 이후 프레임의 해당 트랙 상태에 반영된다. +5. 처리 결과를 원래 WebRTC 트랙으로 반환한다. -`GET /health`에서 장치, ONNX provider, 큐 깊이, 배치 수, 평균 배치 크기와 평균 -처리 지연을 확인할 수 있다. 얼굴 대조 수와 지연, YOLO 랜드마크 사용 수 및 YuNet -폴백 수도 별도 지표로 제공한다. +`GET /health`에서 장치, ONNX provider, 프레임 큐와 식별 큐 깊이, 배치 수, 평균 +배치 크기와 평균 처리 지연을 확인할 수 있다. 얼굴 대조 수와 지연, YOLO 랜드마크 +사용 수, 랜드마크 누락 수, 식별 큐 포화로 드롭된 요청 수도 별도 지표로 제공한다. +`identity_yunet_fallback_faces`는 이전 클라이언트와의 호환성을 위해 남아 있지만, +YuNet fallback은 더 이상 사용하지 않으므로 항상 `0`이다. -한 프레임에서 새로 감지된 모든 얼굴은 ROI Align과 SFace의 단일 CUDA 배치로 -즉시 대조한다. 본인 트랙은 `AI_PRIVACY_IDENTITY_RECHECK_INTERVAL`, 비본인 트랙은 더 긴 -`AI_PRIVACY_IDENTITY_NEGATIVE_RECHECK_INTERVAL` 주기로 재확인한다. +본인 트랙은 `AI_PRIVACY_IDENTITY_RECHECK_INTERVAL`, 비본인 트랙은 더 긴 +`AI_PRIVACY_IDENTITY_NEGATIVE_RECHECK_INTERVAL` 주기로 재확인한다. 새 얼굴이 한 번에 +많이 들어오면 현재 프레임은 기존 추적 상태로 바로 처리하고, 식별 워커가 처리한 +결과가 다음 프레임들에 반영된다. + +### Reference face API + +`/reference-face` API는 `client_id`별로 블러 제외 기준 얼굴을 분리한다. +`client_id`는 쿼리, `multipart/form-data` 필드, 또는 `X-Client-ID` 헤더로 전달할 수 +있다. 값이 없으면 기존 클라이언트 호환을 위해 `default` 클라이언트를 사용한다. + +```text +POST /reference-face +GET /reference-face +DELETE /reference-face +DELETE /reference-face/{face_id} +``` + +`POST /reference-face`는 JPEG, PNG, WebP 이미지를 최대 20장까지 받는다. 기존 단일 +파일 필드 `image`만 보내면 기존 동작처럼 해당 클라이언트의 참조 얼굴을 교체한다. +새 다중 파일 필드 `images`를 보내면 해당 클라이언트에 얼굴을 추가한다. 응답에는 +`client_id`, `count`, `faces[].face_id`, `faces[].registered_at`이 포함된다. + +`DELETE /reference-face`는 해당 클라이언트의 참조 얼굴 전체를 삭제하고, +`DELETE /reference-face/{face_id}`는 해당 클라이언트에서 지정한 한 장만 삭제한다. +참조 이미지는 `data/reference_faces/{client_id}/{face_id}{ext}` 형식으로 저장된다. +예전 단일 파일 `data/reference_face.jpg`는 `default` 클라이언트 조회 시에만 +호환용으로 읽는다. + +WebRTC 세션은 `POST /sessions` 요청의 `metadata.client_id`로 참조 얼굴을 선택한다. +브라우저 테스트 클라이언트는 이 값을 자동으로 넣는다. API 클라이언트가 세션을 +직접 생성할 때 `metadata.client_id`를 생략하면 `default` 클라이언트의 참조 얼굴을 +사용한다. ### NVIDIA RTX 3090 배포 호스트에 NVIDIA 드라이버와 NVIDIA Container Toolkit을 설치한 뒤 `.env.example`을 -기준으로 `.env`를 구성한다. 기본 운영값은 `cuda:0`, FP16, 최대 배치 8개이며 참조 -얼굴 파일은 실제 서비스 대상 이미지로 교체해야 한다. +기준으로 `.env`를 구성한다. 기본 운영값은 `cuda:0`, FP16, 최대 배치 8개다. +고정 참조 얼굴을 환경 변수로 넣고 싶으면 `AI_PRIVACY_ME_IMAGE_PATH`를 설정할 수 +있지만, 일반적인 운영 흐름은 클라이언트가 `/reference-face` API로 자기 +`client_id`의 참조 얼굴을 등록하는 방식이다. ```bash docker build -t innolive-server . @@ -88,8 +125,8 @@ https://innolive.duckdns.org/client/ Use **Start WebRTC** to create a session, open the camera, send a WebRTC offer over `/signaling`, trickle local ICE candidates, and receive the server's processed video track. The page also exposes health check, session create/list, -session polling, deletion, signaling error probing, connection timing, and media -state diagnostics. +session polling, deletion, signaling error probing, connection timing, media +state diagnostics, and client-scoped reference face registration/deletion. ## WebRTC signaling diff --git a/app/api/routes/reference_face.py b/app/api/routes/reference_face.py index 779720b..8e949f1 100644 --- a/app/api/routes/reference_face.py +++ b/app/api/routes/reference_face.py @@ -1,21 +1,34 @@ """촬영 대상자 기준 얼굴 등록·조회·삭제 API.""" +from typing import Annotated + from fastapi import APIRouter, UploadFile, status +from fastapi import File, Form, Header, Query from starlette.concurrency import run_in_threadpool from app.core.config import get_settings from app.core.exceptions import BadRequestError from app.schemas.reference_face import ReferenceFaceStatusResponse from app.services.ai.identity import ( - delete_reference_face, + ReferenceFaceInfo, + ReferenceFaceUpload, + delete_reference_face_by_id, + delete_reference_faces, get_reference_face_info, - register_reference_face, + normalize_reference_client_id, + register_reference_faces, ) router = APIRouter(tags=["reference-face"]) _ALLOWED_CONTENT_TYPES = {"image/jpeg", "image/png", "image/webp"} _MAX_IMAGE_SIZE = 10 * 1024 * 1024 # 10 MB +_MAX_IMAGE_COUNT = 20 +_CONTENT_TYPE_EXTENSIONS = { + "image/jpeg": ".jpg", + "image/png": ".png", + "image/webp": ".webp", +} @router.post( @@ -26,7 +39,11 @@ description="촬영 대상자의 얼굴 이미지를 업로드하여 블러 제외 대상으로 등록합니다.", ) async def post_reference_face( - image: UploadFile, + image: Annotated[UploadFile | None, File()] = None, + images: Annotated[list[UploadFile] | None, File()] = None, + form_client_id: Annotated[str | None, Form(alias="client_id")] = None, + query_client_id: Annotated[str | None, Query(alias="client_id")] = None, + x_client_id: Annotated[str | None, Header(alias="X-Client-ID")] = None, ) -> ReferenceFaceStatusResponse: settings = get_settings() if not settings.ai_privacy_enabled: @@ -35,31 +52,21 @@ async def post_reference_face( details={"reason": "ai_disabled"}, ) - if image.content_type not in _ALLOWED_CONTENT_TYPES: - raise BadRequestError( - "지원하지 않는 이미지 형식입니다. JPEG, PNG, WebP만 허용됩니다.", - details={"content_type": image.content_type}, - ) + client_id = _resolve_client_id(query_client_id, form_client_id, x_client_id) + uploads = _collect_uploads(image=image, images=images) + reference_uploads = [] + for upload in uploads: + reference_uploads.append(await _read_reference_upload(upload)) - image_data = await image.read() - if not image_data: - raise BadRequestError( - "빈 이미지 파일은 업로드할 수 없습니다.", - details={"reason": "empty_file"}, - ) - if len(image_data) > _MAX_IMAGE_SIZE: - raise BadRequestError( - "이미지 크기가 10MB를 초과합니다.", - details={"reason": "file_too_large", "max_bytes": _MAX_IMAGE_SIZE}, - ) - info = await run_in_threadpool(register_reference_face, image_data) - - return ReferenceFaceStatusResponse( - registered=info.registered, - source=info.source, - registered_at=info.registered_at, + info = await run_in_threadpool( + register_reference_faces, + reference_uploads, + client_id=client_id, + replace_existing=image is not None and not images, ) + return _status_response(info) + @router.get( "/reference-face", @@ -67,13 +74,12 @@ async def post_reference_face( summary="기준 얼굴 등록 상태 조회", description="현재 기준 얼굴의 등록 상태를 조회합니다.", ) -async def get_reference_face() -> ReferenceFaceStatusResponse: - info = get_reference_face_info() - return ReferenceFaceStatusResponse( - registered=info.registered, - source=info.source, - registered_at=info.registered_at, - ) +async def get_reference_face( + client_id: Annotated[str | None, Query(alias="client_id")] = None, + x_client_id: Annotated[str | None, Header(alias="X-Client-ID")] = None, +) -> ReferenceFaceStatusResponse: + info = get_reference_face_info(_resolve_client_id(client_id, x_client_id)) + return _status_response(info) @router.delete( @@ -82,7 +88,34 @@ async def get_reference_face() -> ReferenceFaceStatusResponse: summary="기준 얼굴 등록 해제", description="기준 얼굴 등록을 해제합니다. 모든 얼굴이 블러 처리됩니다.", ) -async def remove_reference_face() -> None: +async def remove_reference_face( + client_id: Annotated[str | None, Query(alias="client_id")] = None, + x_client_id: Annotated[str | None, Header(alias="X-Client-ID")] = None, +) -> None: + settings = get_settings() + if not settings.ai_privacy_enabled: + raise BadRequestError( + "AI 프라이버시 기능이 비활성화 상태입니다.", + details={"reason": "ai_disabled"}, + ) + + await run_in_threadpool( + delete_reference_faces, + client_id=_resolve_client_id(client_id, x_client_id), + ) + + +@router.delete( + "/reference-face/{face_id}", + status_code=status.HTTP_204_NO_CONTENT, + summary="개별 기준 얼굴 등록 해제", + description="face_id로 지정한 기준 얼굴 한 장만 블러 제외 대상에서 제거합니다.", +) +async def remove_reference_face_by_id( + face_id: str, + client_id: Annotated[str | None, Query(alias="client_id")] = None, + x_client_id: Annotated[str | None, Header(alias="X-Client-ID")] = None, +) -> None: settings = get_settings() if not settings.ai_privacy_enabled: raise BadRequestError( @@ -90,4 +123,86 @@ async def remove_reference_face() -> None: details={"reason": "ai_disabled"}, ) - await run_in_threadpool(delete_reference_face) + await run_in_threadpool( + delete_reference_face_by_id, + face_id, + client_id=_resolve_client_id(client_id, x_client_id), + ) + + +def _resolve_client_id(*values: str | None) -> str: + for value in values: + if value is not None and value.strip(): + return normalize_reference_client_id(value) + return normalize_reference_client_id(None) + + +def _collect_uploads( + *, + image: UploadFile | None, + images: list[UploadFile] | None, +) -> list[UploadFile]: + uploads = [] + if image is not None: + uploads.append(image) + if images: + uploads.extend(images) + if not uploads: + raise BadRequestError( + "업로드할 이미지가 없습니다.", + details={"reason": "empty_upload"}, + ) + if len(uploads) > _MAX_IMAGE_COUNT: + raise BadRequestError( + "한 번에 등록할 수 있는 이미지는 최대 20장입니다.", + details={"reason": "too_many_files", "max_files": _MAX_IMAGE_COUNT}, + ) + return uploads + + +async def _read_reference_upload(upload: UploadFile) -> ReferenceFaceUpload: + if upload.content_type not in _ALLOWED_CONTENT_TYPES: + raise BadRequestError( + "지원하지 않는 이미지 형식입니다. JPEG, PNG, WebP만 허용됩니다.", + details={ + "content_type": upload.content_type, + "filename": upload.filename, + }, + ) + + image_data = await upload.read() + if not image_data: + raise BadRequestError( + "빈 이미지 파일은 업로드할 수 없습니다.", + details={"reason": "empty_file", "filename": upload.filename}, + ) + if len(image_data) > _MAX_IMAGE_SIZE: + raise BadRequestError( + "이미지 크기가 10MB를 초과합니다.", + details={ + "reason": "file_too_large", + "max_bytes": _MAX_IMAGE_SIZE, + "filename": upload.filename, + }, + ) + return ReferenceFaceUpload( + image_data=image_data, + extension=_CONTENT_TYPE_EXTENSIONS[upload.content_type], + ) + + +def _status_response(info: ReferenceFaceInfo) -> ReferenceFaceStatusResponse: + return ReferenceFaceStatusResponse( + registered=info.registered, + source=info.source, + registered_at=info.registered_at, + client_id=info.client_id, + count=info.count, + faces=[ + { + "face_id": face.face_id, + "registered_at": face.registered_at, + } + for face in info.faces + ], + ) diff --git a/app/core/config.py b/app/core/config.py index 119622d..3c7f13d 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -27,10 +27,7 @@ class Settings(BaseSettings): ai_privacy_identity_recheck_interval: int = 12 ai_privacy_identity_negative_recheck_interval: int = 90 ai_privacy_me_image_path: str | None = None - ai_privacy_face_model_path: str = "models/yolov12n-face.pt" - ai_privacy_face_detection_model_path: str = ( - "models/face_detection_yunet_2023mar.onnx" - ) + ai_privacy_face_model_path: str = "models/yolo11s-pose_widerface.pt" ai_privacy_face_recognition_model_path: str = ( "models/face_recognition_sface_2021dec.onnx" ) diff --git a/app/schemas/reference_face.py b/app/schemas/reference_face.py index 2351c10..03c922e 100644 --- a/app/schemas/reference_face.py +++ b/app/schemas/reference_face.py @@ -3,10 +3,18 @@ from datetime import datetime from typing import Literal -from pydantic import BaseModel +from pydantic import BaseModel, Field + + +class ReferenceFaceImageResponse(BaseModel): + face_id: str + registered_at: datetime class ReferenceFaceStatusResponse(BaseModel): registered: bool source: Literal["api", "env"] | None = None registered_at: datetime | None = None + client_id: str = "default" + count: int = 0 + faces: list[ReferenceFaceImageResponse] = Field(default_factory=list) diff --git a/app/services/ai/filter.py b/app/services/ai/filter.py index b37c807..8ed83c3 100644 --- a/app/services/ai/filter.py +++ b/app/services/ai/filter.py @@ -3,6 +3,7 @@ import logging from pathlib import Path import threading +from collections.abc import Sequence from typing import TYPE_CHECKING, Any from app.core.config import Settings, get_settings @@ -36,20 +37,45 @@ def _asset_path(value: str | Path) -> Path: return path if path.is_absolute() else _PROJECT_ROOT / path -def build_privacy_config(settings: Settings) -> PrivacyBlurConfig: - from app.services.ai.identity import REFERENCE_FACE_PATH +def build_privacy_config( + settings: Settings, + *, + client_id: str | None = None, + reference_paths: Sequence[Path] | None = None, +) -> PrivacyBlurConfig: + from app.services.ai.identity import ( + DEFAULT_REFERENCE_CLIENT_ID, + get_reference_face_paths, + normalize_reference_client_id, + ) - api_reference = _asset_path(REFERENCE_FACE_PATH) + normalized_client_id = normalize_reference_client_id(client_id) + api_references = ( + [_asset_path(path) for path in reference_paths] + if reference_paths is not None + else [_asset_path(path) for path in get_reference_face_paths(normalized_client_id)] + ) environment_reference = ( _asset_path(settings.ai_privacy_me_image_path) if settings.ai_privacy_me_image_path else None ) - reference_path = api_reference if api_reference.exists() else environment_reference - identity_enabled = api_reference.exists() or ( + reference_paths_for_identity = ( + api_references + if api_references + else [environment_reference] + if environment_reference is not None + else [] + ) + identity_enabled = bool(api_references) or ( settings.ai_privacy_enable_identity_exclusion and environment_reference is not None ) + me_image_path = ( + reference_paths_for_identity[0] + if reference_paths_for_identity + else _PROJECT_ROOT / "me.jpeg" + ) return PrivacyBlurConfig( face_model_path=_asset_path(settings.ai_privacy_face_model_path), @@ -65,10 +91,9 @@ def build_privacy_config(settings: Settings) -> PrivacyBlurConfig: identity_negative_recheck_interval=( settings.ai_privacy_identity_negative_recheck_interval ), - me_image_path=reference_path or _PROJECT_ROOT / "me.jpeg", - face_detection_model_path=_asset_path( - settings.ai_privacy_face_detection_model_path - ), + me_image_path=me_image_path, + me_image_paths=tuple(reference_paths_for_identity), + identity_client_id=normalized_client_id or DEFAULT_REFERENCE_CLIENT_ID, face_recognition_model_path=_asset_path( settings.ai_privacy_face_recognition_model_path ), @@ -113,16 +138,31 @@ def initialize_privacy_runtime() -> PrivacyBlurConfig | None: return config -def get_privacy_filter() -> FacePrivacyFilter | None: - from app.services.ai.identity import _reference_face_lock +def get_privacy_filter(client_id: str | None = None) -> FacePrivacyFilter | None: + from app.services.ai.identity import ( + _reference_face_lock, + normalize_reference_client_id, + ) + normalized_client_id = normalize_reference_client_id(client_id) with _reference_face_lock, _state_lock: try: config = initialize_privacy_runtime() if config is None or _runtime is None: return None - privacy_filter = FacePrivacyFilter(config=config, runtime=_runtime) + config_client_id = getattr(config, "identity_client_id", None) + if not isinstance(config_client_id, str): + config_client_id = normalized_client_id + stream_config = ( + config + if config_client_id == normalized_client_id + else build_privacy_config( + get_settings(), + client_id=normalized_client_id, + ) + ) + privacy_filter = FacePrivacyFilter(config=stream_config, runtime=_runtime) with _filters_lock: _filters.append(privacy_filter) return privacy_filter @@ -140,16 +180,29 @@ def get_privacy_filter() -> FacePrivacyFilter | None: def prepare_identity_update( - candidate_path: Path, + reference_paths: Path | Sequence[Path], + *, + client_id: str | None = None, ) -> tuple[PrivacyBlurConfig, Any]: """Validate and prepare a reference embedding without changing live streams.""" + from app.services.ai.identity import normalize_reference_client_id + + normalized_client_id = normalize_reference_client_id(client_id) + if isinstance(reference_paths, Path): + normalized_paths = [reference_paths] + else: + normalized_paths = list(reference_paths) + with _state_lock: config = initialize_privacy_runtime() if config is None or _runtime is None: raise RuntimeError("AI privacy runtime is not initialized") + paths = [_asset_path(path) for path in normalized_paths] candidate_config = config.with_overrides( enable_identity_exclusion=True, - me_image_path=_asset_path(candidate_path), + me_image_path=paths[0], + me_image_paths=tuple(paths), + identity_client_id=normalized_client_id, ) prepared = _runtime.prepare_identity(candidate_config) return candidate_config, prepared @@ -158,13 +211,25 @@ def prepare_identity_update( def commit_identity_update( candidate_config: PrivacyBlurConfig, prepared: Any, - final_path: Path, + final_paths: Path | Sequence[Path], + *, + client_id: str | None = None, ) -> None: """Atomically publish a prepared reference to the runtime and all streams.""" global _config, _config_initialized + from app.services.ai.identity import normalize_reference_client_id + + normalized_client_id = normalize_reference_client_id(client_id) + if isinstance(final_paths, Path): + normalized_paths = [final_paths] + else: + normalized_paths = list(final_paths) + paths = [_asset_path(path) for path in normalized_paths] final_config = candidate_config.with_overrides( - me_image_path=_asset_path(final_path), + me_image_path=paths[0], + me_image_paths=tuple(paths), + identity_client_id=normalized_client_id, ) with _state_lock: if _runtime is None: @@ -172,27 +237,64 @@ def commit_identity_update( _runtime.apply_identity(final_config, prepared) _config = final_config _config_initialized = True - with _filters_lock: - snapshot = list(_filters) - for privacy_filter in snapshot: - privacy_filter.replace_config(final_config) + _replace_client_filter_configs( + final_config, + getattr(prepared, "reference_features", None), + client_id=normalized_client_id, + ) -def disable_identity_exclusion() -> None: +def refresh_identity_for_client(client_id: str | None = None) -> None: global _config, _config_initialized + from app.services.ai.identity import normalize_reference_client_id + + normalized_client_id = normalize_reference_client_id(client_id) with _state_lock: - if _config is None or _runtime is None: + if _runtime is None: invalidate_config() return - config = _config.with_overrides(enable_identity_exclusion=False) - _runtime.apply_identity(config, None) + config = build_privacy_config( + get_settings(), + client_id=normalized_client_id, + ) + prepared = ( + _runtime.prepare_identity(config) + if config.enable_identity_exclusion + else None + ) _config = config _config_initialized = True - with _filters_lock: - snapshot = list(_filters) - for privacy_filter in snapshot: - privacy_filter.replace_config(config) + _runtime.apply_identity(config, prepared) + _replace_client_filter_configs( + config, + getattr(prepared, "reference_features", None) if prepared else None, + client_id=normalized_client_id, + ) + + +def disable_identity_exclusion() -> None: + refresh_identity_for_client() + + +def _replace_client_filter_configs( + config: PrivacyBlurConfig, + reference_features: Any, + *, + client_id: str | None, +) -> None: + from app.services.ai.identity import normalize_reference_client_id + + normalized_client_id = normalize_reference_client_id(client_id) + with _filters_lock: + snapshot = list(_filters) + for privacy_filter in snapshot: + filter_client_id = getattr(privacy_filter.config, "identity_client_id", None) + if not isinstance(filter_client_id, str): + filter_client_id = normalized_client_id + if filter_client_id != normalized_client_id: + continue + privacy_filter.replace_config(config, reference_features) def invalidate_config() -> None: diff --git a/app/services/ai/identity.py b/app/services/ai/identity.py index e7521b7..cdc54ce 100644 --- a/app/services/ai/identity.py +++ b/app/services/ai/identity.py @@ -4,19 +4,43 @@ import logging import os +import re from dataclasses import dataclass from datetime import UTC, datetime +from pathlib import Path import threading +from typing import TYPE_CHECKING from uuid import uuid4 -from app.core.exceptions import BadRequestError +from app.core.exceptions import BadRequestError, NotFoundError from app.services.ai.filter import _MODELS_DIR +if TYPE_CHECKING: + from privacy_blur import PrivacyBlurConfig + logger = logging.getLogger(__name__) REFERENCE_FACE_DIR = _MODELS_DIR.parent / "data" -REFERENCE_FACE_PATH = REFERENCE_FACE_DIR / "reference_face.jpg" +REFERENCE_FACE_ROOT = REFERENCE_FACE_DIR / "reference_faces" +LEGACY_REFERENCE_FACE_PATH = REFERENCE_FACE_DIR / "reference_face.jpg" +REFERENCE_FACE_PATH = LEGACY_REFERENCE_FACE_PATH +DEFAULT_REFERENCE_CLIENT_ID = "default" _reference_face_lock = threading.Lock() +_CLIENT_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$") +_REFERENCE_FACE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp"} + + +@dataclass(frozen=True, slots=True) +class ReferenceFaceUpload: + image_data: bytes + extension: str = ".jpg" + + +@dataclass(frozen=True, slots=True) +class ReferenceFaceImageInfo: + face_id: str + path: Path + registered_at: datetime @dataclass(frozen=True, slots=True) @@ -24,24 +48,106 @@ class ReferenceFaceInfo: registered: bool source: str | None = None registered_at: datetime | None = None + client_id: str = DEFAULT_REFERENCE_CLIENT_ID + count: int = 0 + faces: tuple[ReferenceFaceImageInfo, ...] = () -def register_reference_face(image_data: bytes) -> ReferenceFaceInfo: - """기준 얼굴을 검증한 뒤 원자적으로 저장하고 공유 런타임에 반영한다.""" +def normalize_reference_client_id(client_id: str | None) -> str: + """클라이언트별 저장 디렉터리로 안전하게 쓸 수 있는 ID를 반환한다.""" + normalized = (client_id or DEFAULT_REFERENCE_CLIENT_ID).strip() + if not normalized: + normalized = DEFAULT_REFERENCE_CLIENT_ID + if not _CLIENT_ID_PATTERN.fullmatch(normalized): + raise BadRequestError( + "client_id는 영문, 숫자, '.', '_' 또는 '-'로 이루어진 1~64자여야 합니다.", + details={"reason": "invalid_client_id", "client_id": client_id}, + ) + return normalized + + +def reference_face_client_dir(client_id: str | None) -> Path: + return REFERENCE_FACE_ROOT / normalize_reference_client_id(client_id) + + +def get_reference_face_paths(client_id: str | None = None) -> list[Path]: + """클라이언트에 등록된 API 기준 얼굴 이미지 경로를 반환한다.""" + normalized_client_id = normalize_reference_client_id(client_id) + paths: list[Path] = [] + client_dir = reference_face_client_dir(normalized_client_id) + if client_dir.exists(): + paths.extend( + path + for path in sorted(client_dir.iterdir()) + if path.is_file() + and not path.name.startswith(".") + and path.suffix.lower() in _REFERENCE_FACE_SUFFIXES + ) + if ( + normalized_client_id == DEFAULT_REFERENCE_CLIENT_ID + and LEGACY_REFERENCE_FACE_PATH.exists() + ): + paths.insert(0, LEGACY_REFERENCE_FACE_PATH) + return paths + + +def register_reference_face( + image_data: bytes, + *, + client_id: str | None = None, +) -> ReferenceFaceInfo: + """단일 기준 얼굴 등록 호환 래퍼.""" + return register_reference_faces( + [ReferenceFaceUpload(image_data=image_data)], + client_id=client_id, + replace_existing=True, + ) + + +def register_reference_faces( + uploads: list[ReferenceFaceUpload], + *, + client_id: str | None = None, + replace_existing: bool = False, +) -> ReferenceFaceInfo: + """기준 얼굴들을 검증한 뒤 원자적으로 추가 저장하고 공유 런타임에 반영한다.""" import app.services.ai.filter as filter_module + normalized_client_id = normalize_reference_client_id(client_id) + if not uploads: + raise BadRequestError( + "업로드할 이미지가 없습니다.", + details={"reason": "empty_upload"}, + ) + with _reference_face_lock: - REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - candidate_path = REFERENCE_FACE_DIR / f".reference_face.{uuid4().hex}.tmp" - candidate_path.write_bytes(image_data) + client_dir = reference_face_client_dir(normalized_client_id) + client_dir.mkdir(parents=True, exist_ok=True) + candidate_pairs: list[tuple[Path, Path]] = [] + for upload in uploads: + extension = _normalize_extension(upload.extension) + face_id = uuid4().hex + candidate_path = client_dir / f".reference_face.{face_id}.tmp" + final_path = client_dir / f"{face_id}{extension}" + candidate_path.write_bytes(upload.image_data) + candidate_pairs.append((candidate_path, final_path)) + prepared_update = None + existing_paths: list[Path] = [] try: + existing_paths = get_reference_face_paths(normalized_client_id) if filter_module._runtime is None: - _validate_image(image_data) + for upload in uploads: + _validate_image(upload.image_data) else: try: + reference_paths = [ + *([] if replace_existing else existing_paths), + *(candidate for candidate, _ in candidate_pairs), + ] prepared_update = filter_module.prepare_identity_update( - candidate_path + reference_paths, + client_id=normalized_client_id, ) except _invalid_reference_image_error() as exc: raise BadRequestError( @@ -54,62 +160,158 @@ def register_reference_face(image_data: bytes) -> ReferenceFaceInfo: details={"reason": "face_not_detected"}, ) from exc - os.replace(candidate_path, REFERENCE_FACE_PATH) + for candidate_path, final_path in candidate_pairs: + os.replace(candidate_path, final_path) + if replace_existing: + for path in existing_paths: + path.unlink(missing_ok=True) + if prepared_update is None: filter_module.invalidate_config() else: candidate_config, prepared = prepared_update + final_reference_paths = [ + *(final for _, final in candidate_pairs), + ] + if not replace_existing: + final_reference_paths = [ + *existing_paths, + *final_reference_paths, + ] filter_module.commit_identity_update( candidate_config, prepared, - REFERENCE_FACE_PATH, + final_reference_paths, + client_id=normalized_client_id, ) finally: - candidate_path.unlink(missing_ok=True) + for candidate_path, _ in candidate_pairs: + candidate_path.unlink(missing_ok=True) - registered_at = datetime.fromtimestamp( - REFERENCE_FACE_PATH.stat().st_mtime, - tz=UTC, - ) + info = get_reference_face_info(normalized_client_id) - logger.info("Registered reference face: source=api") - return ReferenceFaceInfo( - registered=True, - source="api", - registered_at=registered_at, + logger.info( + "Registered reference faces: source=api client_id=%s added=%d total=%d replace=%s", + normalized_client_id, + len(uploads), + info.count, + replace_existing, ) + return info -def get_reference_face_info() -> ReferenceFaceInfo: +def get_reference_face_info(client_id: str | None = None) -> ReferenceFaceInfo: """현재 기준 얼굴 등록 상태를 반환한다.""" from app.core.config import get_settings - if REFERENCE_FACE_PATH.exists(): - mtime = REFERENCE_FACE_PATH.stat().st_mtime + normalized_client_id = normalize_reference_client_id(client_id) + api_paths = get_reference_face_paths(normalized_client_id) + if api_paths: + faces = tuple(_face_info_from_path(path) for path in api_paths) return ReferenceFaceInfo( registered=True, source="api", - registered_at=datetime.fromtimestamp(mtime, tz=UTC), + registered_at=max(face.registered_at for face in faces), + client_id=normalized_client_id, + count=len(faces), + faces=faces, ) settings = get_settings() if settings.ai_privacy_me_image_path: - return ReferenceFaceInfo(registered=True, source="env") + return ReferenceFaceInfo( + registered=True, + source="env", + client_id=normalized_client_id, + count=1, + ) + + return ReferenceFaceInfo(registered=False, client_id=normalized_client_id) - return ReferenceFaceInfo(registered=False) +def delete_reference_face(*, client_id: str | None = None) -> None: + """단일 기준 얼굴 삭제 호환 래퍼.""" + delete_reference_faces(client_id=client_id) -def delete_reference_face() -> None: + +def delete_reference_face_by_id( + face_id: str, + *, + client_id: str | None = None, +) -> ReferenceFaceInfo: + """클라이언트에 등록된 기준 얼굴 중 지정한 face_id만 삭제한다.""" + import app.services.ai.filter as filter_module + + normalized_client_id = normalize_reference_client_id(client_id) + with _reference_face_lock: + paths = get_reference_face_paths(normalized_client_id) + target = next((path for path in paths if path.stem == face_id), None) + if target is None: + raise NotFoundError( + "해당 기준 얼굴을 찾을 수 없습니다.", + details={ + "reason": "reference_face_not_found", + "client_id": normalized_client_id, + "face_id": face_id, + }, + ) + + target.unlink(missing_ok=True) + _remove_empty_reference_client_dir(normalized_client_id) + if filter_module._runtime is None: + filter_module.invalidate_config() + else: + filter_module.refresh_identity_for_client(normalized_client_id) + info = get_reference_face_info(normalized_client_id) + + logger.info( + "Deleted reference face: client_id=%s face_id=%s remaining=%d", + normalized_client_id, + face_id, + info.count, + ) + return info + + +def delete_reference_faces(*, client_id: str | None = None) -> None: """기준 얼굴 등록을 해제하고 모든 얼굴을 블러 대상으로 전환한다.""" import app.services.ai.filter as filter_module + normalized_client_id = normalize_reference_client_id(client_id) with _reference_face_lock: - REFERENCE_FACE_PATH.unlink(missing_ok=True) + for path in get_reference_face_paths(normalized_client_id): + path.unlink(missing_ok=True) + _remove_empty_reference_client_dir(normalized_client_id) if filter_module._runtime is None: filter_module.invalidate_config() else: - filter_module.disable_identity_exclusion() - logger.info("Deleted reference face") + filter_module.refresh_identity_for_client(normalized_client_id) + logger.info("Deleted reference faces: client_id=%s", normalized_client_id) + + +def _remove_empty_reference_client_dir(client_id: str | None) -> None: + client_dir = reference_face_client_dir(client_id) + if client_dir.exists() and not any(client_dir.iterdir()): + client_dir.rmdir() + + +def _normalize_extension(extension: str) -> str: + normalized = extension.lower() + if not normalized.startswith("."): + normalized = f".{normalized}" + if normalized == ".jpeg": + return ".jpg" + if normalized not in _REFERENCE_FACE_SUFFIXES: + return ".jpg" + return normalized + + +def _face_info_from_path(path: Path) -> ReferenceFaceImageInfo: + return ReferenceFaceImageInfo( + face_id=path.stem, + path=path, + registered_at=datetime.fromtimestamp(path.stat().st_mtime, tz=UTC), + ) def _invalid_reference_image_error() -> type[Exception]: @@ -128,7 +330,10 @@ def _validate_image(image_data: bytes) -> None: """이미지 디코딩 및 얼굴 감지 가능 여부를 검증한다.""" import cv2 import numpy as np - from privacy_blur.face_detection import detect_best_yunet_face + from ultralytics import YOLO + + from app.core.config import get_settings + from privacy_blur import PrivacyBlurConfig img_array = np.frombuffer(image_data, dtype=np.uint8) image = cv2.imdecode(img_array, cv2.IMREAD_COLOR) @@ -138,15 +343,62 @@ def _validate_image(image_data: bytes) -> None: details={"reason": "invalid_image"}, ) - detection_model = _MODELS_DIR / "face_detection_yunet_2023mar.onnx" - if not detection_model.exists(): - return - - detector = cv2.FaceDetectorYN.create( - str(detection_model), "", (320, 320), 0.55, 0.3, 5000, + settings = get_settings() + model_path = Path(settings.ai_privacy_face_model_path).expanduser() + if not model_path.is_absolute(): + model_path = _MODELS_DIR.parent / model_path + if not model_path.exists(): + raise RuntimeError(f"AI privacy face model not found: {model_path}") + + config = PrivacyBlurConfig( + face_model_path=model_path, + face_imgsz=settings.ai_privacy_face_imgsz, + enable_identity_exclusion=False, ) - if detect_best_yunet_face(detector, image) is None: + model = YOLO(str(config.face_model_path)) + results = model.predict( + image, + imgsz=config.face_imgsz, + conf=config.face_conf_threshold, + iou=config.face_iou_threshold, + max_det=config.face_max_detections, + verbose=False, + device="cpu", + half=False, + ) + if not results or not _has_pose_face(results[0], config): raise BadRequestError( "업로드한 이미지에서 얼굴을 감지할 수 없습니다.", details={"reason": "face_not_detected"}, ) + + +def _has_pose_face(result: object, config: "PrivacyBlurConfig") -> bool: + import numpy as np + import torch + + boxes = getattr(result, "boxes", None) + keypoints = getattr(result, "keypoints", None) + if boxes is None or keypoints is None or not len(boxes) or not len(keypoints): + return False + + keypoint_xy = torch.as_tensor(keypoints.xy).detach().float().cpu().numpy() + keypoint_conf = ( + torch.as_tensor(keypoints.conf).detach().float().cpu().numpy() + if keypoints.conf is not None + else None + ) + for index, landmarks in enumerate(keypoint_xy): + confidence_ok = ( + keypoint_conf is None + or index >= len(keypoint_conf) + or bool(np.all(keypoint_conf[index] >= config.identity_detector_confidence)) + ) + if ( + landmarks.shape == (5, 2) + and np.isfinite(landmarks).all() + and float(np.min(np.ptp(landmarks, axis=0))) >= 1.0 + and confidence_ok + ): + return True + return False diff --git a/app/services/ai/track.py b/app/services/ai/track.py index 73d691b..215e386 100644 --- a/app/services/ai/track.py +++ b/app/services/ai/track.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import logging from typing import TYPE_CHECKING @@ -34,7 +35,7 @@ def is_fallback_active(self) -> bool: return self._fallback async def recv(self): - frame = await self._source.recv() + frame = await _recv_latest_frame(self._source) if self._fallback: return _blackout_frame(frame) try: @@ -67,6 +68,35 @@ def stop(self) -> None: pass +async def _recv_latest_frame(source: MediaStreamTrack) -> VideoFrame: + frame = await source.recv() + queue = getattr(source, "_queue", None) + if not isinstance(queue, asyncio.Queue): + return frame + + dropped = 0 + while True: + try: + latest = queue.get_nowait() + except asyncio.QueueEmpty: + break + if latest is None: + try: + queue.put_nowait(None) + except asyncio.QueueFull: + pass + break + frame = latest + dropped += 1 + + if dropped: + logger.debug( + "Dropped stale source frames before AI processing: count=%d", + dropped, + ) + return frame + + def _blackout_frame(frame: VideoFrame) -> VideoFrame: image = frame.to_ndarray(format="bgr24") black = VideoFrame.from_ndarray(np.zeros_like(image), format="bgr24") diff --git a/app/services/sessions/manager.py b/app/services/sessions/manager.py index bef5cfb..90f7ce3 100644 --- a/app/services/sessions/manager.py +++ b/app/services/sessions/manager.py @@ -431,7 +431,11 @@ async def on_track_ended() -> None: ) session.raw_video_track = track - wrapped = _wrap_with_ai(track, session.session_id) + wrapped = _wrap_with_ai( + track, + session.session_id, + client_id=_session_client_id(session), + ) session.processed_video_track = wrapped stream_is_active = ( session.stream.status == "ready" @@ -872,11 +876,13 @@ def _track_id(track: MediaStreamTrack) -> str | None: def _wrap_with_ai( track: MediaStreamTrack, session_id: str, + *, + client_id: str | None = None, ) -> MediaStreamTrack: from app.core.config import get_settings from app.services.ai.filter import get_privacy_filter - privacy_filter = get_privacy_filter() + privacy_filter = get_privacy_filter(client_id=client_id) if privacy_filter is None and not get_settings().ai_privacy_enabled: return track @@ -889,6 +895,10 @@ def _wrap_with_ai( ) +def _session_client_id(session: LiveSession) -> str | None: + return session.metadata.get("client_id") + + def _track_to_dict(track: MediaStreamTrack | None) -> dict[str, str | None] | None: if track is None: return None diff --git a/app/services/webrtc/track_tee.py b/app/services/webrtc/track_tee.py index 84ef297..dfe9cbd 100644 --- a/app/services/webrtc/track_tee.py +++ b/app/services/webrtc/track_tee.py @@ -24,6 +24,18 @@ async def recv(self): raise MediaStreamError frame = await self._queue.get() + while True: + try: + latest = self._queue.get_nowait() + except asyncio.QueueEmpty: + break + if latest is _END_OF_STREAM: + try: + self._queue.put_nowait(_END_OF_STREAM) + except asyncio.QueueFull: + pass + break + frame = latest if frame is _END_OF_STREAM: self.stop() raise MediaStreamError diff --git a/app/static/client/app.js b/app/static/client/app.js index db9d406..b347b37 100644 --- a/app/static/client/app.js +++ b/app/static/client/app.js @@ -1,5 +1,6 @@ const DEFAULT_SERVER_URL = "https://innolive.duckdns.org"; const SERVER_STORAGE_KEY = "inno-live-client.server-url"; +const CLIENT_ID_STORAGE_KEY = "inno-live-client.client-id"; const MAX_LOG_ITEMS = 120; const DEFAULT_ICE_SERVERS = [{ urls: "stun:stun.l.google.com:19302" }]; @@ -60,6 +61,7 @@ function bindElements() { "servedClientLink", "referenceFaceState", "referenceFaceDetail", + "referenceFaceList", "referenceFaceInput", "referenceFacePreview", "referenceFacePreviewText", @@ -174,6 +176,22 @@ function persistServerUrl() { } } +function getClientId() { + try { + const stored = localStorage.getItem(CLIENT_ID_STORAGE_KEY); + if (stored) { + return stored; + } + const generated = crypto.randomUUID + ? crypto.randomUUID() + : `client-${Date.now()}-${Math.random().toString(16).slice(2)}`; + localStorage.setItem(CLIENT_ID_STORAGE_KEY, generated); + return generated; + } catch { + return "web-client"; + } +} + function normalizeServerUrl(rawValue) { let value = String(rawValue || "").trim(); if (!value) { @@ -189,6 +207,16 @@ function apiUrl(path) { return new URL(path, normalizeServerUrl(els.serverUrl.value)).href; } +function referenceFaceApiPath() { + const params = new URLSearchParams({ client_id: getClientId() }); + return `/reference-face?${params.toString()}`; +} + +function referenceFaceItemApiPath(faceId) { + const params = new URLSearchParams({ client_id: getClientId() }); + return `/reference-face/${encodeURIComponent(faceId)}?${params.toString()}`; +} + function signalingUrl() { const url = new URL("/signaling", normalizeServerUrl(els.serverUrl.value)); url.protocol = url.protocol === "https:" ? "wss:" : "ws:"; @@ -238,7 +266,7 @@ function parseJsonOrText(text) { async function refreshReferenceFace({ quiet = false } = {}) { setPill(els.referenceFaceState, "확인 중", "warn"); try { - const status = await apiFetch("/reference-face"); + const status = await apiFetch(referenceFaceApiPath()); state.referenceFaceSupported = true; state.referenceFace = status; renderReferenceFaceStatus(); @@ -279,7 +307,7 @@ function updateReferenceFacePreview() { if (!file) { els.referenceFacePreview.hidden = true; els.referenceFacePreview.removeAttribute("src"); - els.referenceFacePreviewText.textContent = "JPEG, PNG, WebP · 최대 10MB"; + els.referenceFacePreviewText.textContent = "JPEG, PNG, WebP · 파일당 최대 10MB"; updateButtons(); return; } @@ -288,29 +316,33 @@ function updateReferenceFacePreview() { els.referenceFacePreview.src = state.referenceFacePreviewUrl; els.referenceFacePreview.alt = `${file.name} 미리보기`; els.referenceFacePreview.hidden = false; - els.referenceFacePreviewText.textContent = - `${file.name} · ${formatBytes(file.size)}`; + const files = Array.from(els.referenceFaceInput.files); + els.referenceFacePreviewText.textContent = files.length === 1 + ? `${file.name} · ${formatBytes(file.size)}` + : `${files.length}개 파일 · 첫 파일 ${formatBytes(file.size)}`; updateButtons(); } async function uploadReferenceFace() { - const [file] = els.referenceFaceInput.files; - if (!file) { + const files = Array.from(els.referenceFaceInput.files); + if (!files.length) { return; } await runReferenceFaceBusy(async () => { const form = new FormData(); - form.append("image", file, file.name); - const status = await apiFetch("/reference-face", { + form.append("client_id", getClientId()); + for (const file of files) { + form.append("images", file, file.name); + } + const status = await apiFetch(referenceFaceApiPath(), { method: "POST", body: form, }); state.referenceFace = status; renderReferenceFaceStatus(); logEvent("ok", "Reference face registered", { - name: file.name, - size: file.size, + files: files.map((file) => ({ name: file.name, size: file.size })), status, }); }); @@ -318,7 +350,7 @@ async function uploadReferenceFace() { async function deleteReferenceFace() { await runReferenceFaceBusy(async () => { - await apiFetch("/reference-face", { method: "DELETE" }); + await apiFetch(referenceFaceApiPath(), { method: "DELETE" }); els.referenceFaceInput.value = ""; updateReferenceFacePreview(); await refreshReferenceFace({ quiet: true }); @@ -326,6 +358,14 @@ async function deleteReferenceFace() { }); } +async function deleteReferenceFaceById(faceId) { + await runReferenceFaceBusy(async () => { + await apiFetch(referenceFaceItemApiPath(faceId), { method: "DELETE" }); + await refreshReferenceFace({ quiet: true }); + logEvent("ok", "Reference face removed", { face_id: faceId }); + }); +} + async function runReferenceFaceBusy(task) { if (state.referenceFaceBusy) { return; @@ -348,21 +388,52 @@ function renderReferenceFaceStatus() { setPill(els.referenceFaceState, "미등록", "idle"); els.referenceFaceDetail.textContent = "현재 모든 감지된 얼굴이 블러 처리됩니다."; + renderReferenceFaceList([]); updateButtons(); return; } setPill(els.referenceFaceState, "등록됨", "ok"); const source = status.source === "env" ? "서버 설정" : "업로드"; + const countText = status.count ? `${status.count}장` : "사진 수 정보 없음"; const registeredAt = status.source === "env" ? "클라이언트에서 해제할 수 없음" : status.registered_at ? new Date(status.registered_at).toLocaleString() : "시간 정보 없음"; - els.referenceFaceDetail.textContent = `${source} · ${registeredAt}`; + els.referenceFaceDetail.textContent = `${source} · ${countText} · ${registeredAt}`; + renderReferenceFaceList(status.faces || []); updateButtons(); } +function renderReferenceFaceList(faces) { + els.referenceFaceList.replaceChildren(); + if (!faces.length) { + return; + } + + for (const face of faces) { + const row = document.createElement("div"); + row.className = "reference-face-item"; + + const label = document.createElement("span"); + const registeredAt = face.registered_at + ? new Date(face.registered_at).toLocaleString() + : "시간 정보 없음"; + label.textContent = `${face.face_id.slice(0, 8)} · ${registeredAt}`; + + const button = document.createElement("button"); + button.className = "button button-danger button-small"; + button.type = "button"; + button.textContent = "삭제"; + button.disabled = state.referenceFaceBusy; + button.addEventListener("click", () => void deleteReferenceFaceById(face.face_id)); + + row.append(label, button); + els.referenceFaceList.append(row); + } +} + function formatBytes(bytes) { if (bytes < 1024) { return `${bytes} B`; @@ -420,6 +491,7 @@ function buildSessionMetadata() { const label = els.sessionLabel.value.trim(); const metadata = { client: "inno-live-web-client", + client_id: getClientId(), started_at: new Date().toISOString(), }; if (label) { diff --git a/app/static/client/index.html b/app/static/client/index.html index 8fc6b31..c4294e5 100644 --- a/app/static/client/index.html +++ b/app/static/client/index.html @@ -80,22 +80,24 @@

WebRTC Client

Privacy exception

본인 얼굴은 블러하지 않기

- 얼굴이 선명하게 보이는 본인 사진 한 장을 등록한 뒤 WebRTC를 시작하세요. - 다른 사람의 얼굴은 계속 블러 처리됩니다. + 얼굴이 선명하게 보이는 사진을 등록한 뒤 WebRTC를 시작하세요. + 등록된 사람들의 얼굴은 블러 제외 대상으로 처리됩니다.

확인 중 등록 상태를 불러오는 중입니다.
+
diff --git a/app/static/client/styles.css b/app/static/client/styles.css index 7b798a8..9b6c344 100644 --- a/app/static/client/styles.css +++ b/app/static/client/styles.css @@ -246,6 +246,32 @@ h2 { font-weight: 700; } +.reference-face-list { + display: grid; + gap: 6px; + margin-top: 10px; +} + +.reference-face-item { + display: flex; + min-height: 34px; + align-items: center; + justify-content: space-between; + gap: 10px; + border: 1px solid var(--border); + border-radius: 7px; + padding: 6px 8px; + background: var(--surface-strong); + color: var(--muted); + font-size: 0.82rem; + font-weight: 800; +} + +.reference-face-item > span { + min-width: 0; + overflow-wrap: anywhere; +} + .reference-face-controls { display: grid; gap: 10px; @@ -314,6 +340,12 @@ h2 { padding: 0 14px; } +.button-small { + min-height: 28px; + padding: 0 9px; + font-size: 0.78rem; +} + .button:hover:not(:disabled), .text-button:hover:not(:disabled) { border-color: #b7c4cc; diff --git a/models/face_detection_yunet_2023mar.onnx b/models/face_detection_yunet_2023mar.onnx deleted file mode 100644 index f9beb30..0000000 Binary files a/models/face_detection_yunet_2023mar.onnx and /dev/null differ diff --git a/models/yolo11s-pose_widerface.pt b/models/yolo11s-pose_widerface.pt new file mode 100644 index 0000000..e5aa7a5 Binary files /dev/null and b/models/yolo11s-pose_widerface.pt differ diff --git a/privacy_blur/config.py b/privacy_blur/config.py index 2552dae..c09dfcc 100644 --- a/privacy_blur/config.py +++ b/privacy_blur/config.py @@ -7,7 +7,7 @@ @dataclass(frozen=True, slots=True) class PrivacyBlurConfig: - face_model_path: Path = Path("models/yolov12n-face.pt") + face_model_path: Path = Path("models/yolo11s-pose_widerface.pt") face_imgsz: int = 640 face_conf_threshold: float = 0.22 face_iou_threshold: float = 0.45 @@ -25,11 +25,11 @@ class PrivacyBlurConfig: enable_identity_exclusion: bool = True me_image_path: Path = Path("me.jpeg") - face_detection_model_path: Path = Path("models/face_detection_yunet_2023mar.onnx") + me_image_paths: tuple[Path, ...] = () + identity_client_id: str | None = None face_recognition_model_path: Path = Path( "models/face_recognition_sface_2021dec.onnx" ) - identity_detector_size: int = 320 identity_detector_confidence: float = 0.55 face_match_cosine_threshold: float = 0.40 face_match_lost_cosine_threshold: float = 0.30 @@ -56,7 +56,9 @@ def __post_init__(self) -> None: object.__setattr__(self, "face_model_path", Path(self.face_model_path)) object.__setattr__(self, "me_image_path", Path(self.me_image_path)) object.__setattr__( - self, "face_detection_model_path", Path(self.face_detection_model_path) + self, + "me_image_paths", + tuple(Path(path) for path in self.me_image_paths), ) object.__setattr__( self, "face_recognition_model_path", Path(self.face_recognition_model_path) @@ -70,8 +72,6 @@ def __post_init__(self) -> None: raise ValueError("max_pending_frames must be at least max_batch_size") if self.detect_every_n_frames < 1: raise ValueError("detect_every_n_frames must be at least 1") - if self.identity_detector_size < 32 or self.identity_detector_size % 32: - raise ValueError("identity_detector_size must be a multiple of 32") if self.identity_recheck_interval < 1: raise ValueError("identity_recheck_interval must be at least 1") if self.identity_negative_recheck_interval < self.identity_recheck_interval: @@ -82,3 +82,6 @@ def __post_init__(self) -> None: def with_overrides(self, **kwargs: Any) -> "PrivacyBlurConfig": return replace(self, **kwargs) + + def identity_image_paths(self) -> tuple[Path, ...]: + return self.me_image_paths or (self.me_image_path,) diff --git a/privacy_blur/core.py b/privacy_blur/core.py index 87933f7..4a2c62b 100644 --- a/privacy_blur/core.py +++ b/privacy_blur/core.py @@ -5,6 +5,7 @@ from typing import Any import numpy as np +import torch from .config import PrivacyBlurConfig from .errors import PrivacyBlurNotReadyError, UnsupportedFrameTypeError @@ -65,6 +66,7 @@ class FaceTrack: last_seen_frame: int last_checked_frame: int landmarks: np.ndarray | None = None + identity_pending: bool = False class FacePrivacyEngine: @@ -81,6 +83,10 @@ def __init__( self.next_track_id = 1 self.tracks: list[FaceTrack] = [] self.last_detection_count = 0 + self.reference_features: torch.Tensor | None = None + if self.config.enable_identity_exclusion: + prepared = self.runtime.prepare_identity(self.config) + self.reference_features = prepared.reference_features self._lock = threading.RLock() def _active_tracks(self) -> list[FaceTrack]: @@ -240,11 +246,12 @@ def begin_frame( ) ) - if needs_identity_check: + if needs_identity_check and not track.identity_pending: if ( min(float(xywh[2]), float(xywh[3])) >= self.config.identity_min_face_size ): + track.identity_pending = True identity_checks.append(track) else: track.similarity = -1.0 @@ -271,6 +278,10 @@ def begin_frame( def apply_identity_matches(self, matches: list[tuple[FaceTrack, float]]) -> None: with self._lock: for track, similarity in matches: + track.identity_pending = False + if similarity < 0.0: + track.similarity = similarity + continue previous_is_me = track.is_me track.similarity = similarity track.last_checked_frame = self.frame_index @@ -300,9 +311,20 @@ def reset(self) -> None: self.tracks = [] self.last_detection_count = 0 - def replace_config(self, config: PrivacyBlurConfig) -> None: + def replace_config( + self, + config: PrivacyBlurConfig, + reference_features: torch.Tensor | None = None, + ) -> None: with self._lock: self.config = config + if not config.enable_identity_exclusion: + self.reference_features = None + elif reference_features is not None: + self.reference_features = reference_features + else: + prepared = self.runtime.prepare_identity(config) + self.reference_features = prepared.reference_features self.frame_index = 0 self.next_track_id = 1 self.tracks = [] @@ -323,6 +345,7 @@ def stats(self) -> dict[str, Any]: "similarity": track.similarity, "last_seen_frame": track.last_seen_frame, "last_checked_frame": track.last_checked_frame, + "identity_pending": track.identity_pending, } for track in active_tracks ], @@ -375,11 +398,18 @@ async def apply_async(self, frame: Any) -> Any: def reset(self) -> None: self.engine.reset() - def replace_config(self, config: PrivacyBlurConfig) -> None: + def replace_config( + self, + config: PrivacyBlurConfig, + reference_features: torch.Tensor | None = None, + ) -> None: if self._closed: raise RuntimeError("The privacy filter is closed") self.config = config - self.engine.replace_config(config) + if reference_features is None: + self.engine.replace_config(config) + else: + self.engine.replace_config(config, reference_features) def stats(self) -> dict[str, Any]: return self.engine.stats() diff --git a/privacy_blur/face_detection.py b/privacy_blur/face_detection.py deleted file mode 100644 index 24e2de5..0000000 --- a/privacy_blur/face_detection.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence -from typing import Any - -import cv2 -import numpy as np - - -# YuNet is trained primarily for faces around 10-300 pixels. Reference photos -# from phones are commonly several thousand pixels wide, so detect on bounded -# resolutions and map the selected face back to the original image. -REFERENCE_DETECTION_MAX_SIZES = (640, 320) - - -def select_reference_face(faces: np.ndarray | None) -> np.ndarray | None: - if faces is None or len(faces) == 0: - return None - areas = faces[:, 2] * faces[:, 3] - scores = faces[:, -1] - rank = areas * scores - return faces[int(np.argmax(rank))] - - -def detect_best_yunet_face( - detector: Any, - image: np.ndarray, - *, - max_sizes: Sequence[int] = REFERENCE_DETECTION_MAX_SIZES, -) -> np.ndarray | None: - """Detect a reference face at YuNet-friendly resolutions. - - The returned box and landmarks use coordinates from the original image. - """ - height, width = image.shape[:2] - if width <= 0 or height <= 0: - return None - - previous_size: tuple[int, int] | None = None - for max_size in max_sizes: - scale = min(1.0, max_size / max(width, height)) - input_width = max(1, int(round(width * scale))) - input_height = max(1, int(round(height * scale))) - input_size = (input_width, input_height) - if input_size == previous_size: - continue - previous_size = input_size - - if input_size == (width, height): - detection_image = image - else: - detection_image = cv2.resize( - image, - input_size, - interpolation=cv2.INTER_AREA, - ) - - detector.setInputSize(input_size) - _, faces = detector.detect(detection_image) - face = select_reference_face( - None if faces is None else faces.astype(np.float32) - ) - if face is None: - continue - - if input_size != (width, height): - face = face.copy() - x_scale = input_width / width - y_scale = input_height / height - face[[0, 2, 4, 6, 8, 10, 12]] /= x_scale - face[[1, 3, 5, 7, 9, 11, 13]] /= y_scale - return face - - return None diff --git a/privacy_blur/runtime.py b/privacy_blur/runtime.py index 344ae7a..8cd7f8b 100644 --- a/privacy_blur/runtime.py +++ b/privacy_blur/runtime.py @@ -27,7 +27,6 @@ PrivacyBlurNotReadyError, ReferenceFaceNotDetectedError, ) -from .face_detection import detect_best_yunet_face if TYPE_CHECKING: from .core import FaceDetection, FacePrivacyEngine, FaceTrack @@ -44,11 +43,21 @@ class _FrameRequest: future: Future[np.ndarray] +@dataclass(slots=True) +class _IdentityRequest: + engine: FacePrivacyEngine + track: FaceTrack + frame: np.ndarray + xywh: np.ndarray + landmarks: np.ndarray | None + reference_features: torch.Tensor | None + generation: int + + @dataclass(slots=True) class _PreparedIdentity: - yunet_session: ort.InferenceSession sface_session: ort.InferenceSession - reference_feature: torch.Tensor + reference_features: torch.Tensor def _require_file(path: Path) -> None: @@ -95,46 +104,20 @@ def _remove_initializer_inputs(model: onnx.ModelProto) -> None: model.graph.input.extend(real_inputs) -def _dynamic_onnx_model(path: Path, kind: str, identity_size: int) -> bytes: +def _dynamic_onnx_model(path: Path) -> bytes: model = onnx.load(str(path)) _remove_initializer_inputs(model) - if kind == "sface": - values = ( - list(model.graph.input) - + list(model.graph.output) - + list(model.graph.value_info) - ) - for value in values: - dimensions = value.type.tensor_type.shape.dim - if dimensions and dimensions[0].dim_value == 1: - dimensions[0].ClearField("dim_value") - dimensions[0].dim_param = "batch" - elif kind == "yunet": - dimensions = model.graph.input[0].type.tensor_type.shape.dim - dimensions[0].ClearField("dim_value") - dimensions[0].dim_param = "batch" - dimensions[2].dim_value = identity_size - dimensions[3].dim_value = identity_size - - # The bundled YuNet model contains intermediate value_info shapes inferred - # for its original 640x640 input. ONNX Runtime uses those stale values to - # restore static 6400/1600/400 output dimensions even after the graph - # outputs are marked dynamic. CUDA I/O binding then rejects the correctly - # sized buffers produced for a 320x320 input. Intermediate value_info is - # optional, so discard it and let ONNX Runtime infer it from the new input. - del model.graph.value_info[:] - for value in model.graph.output: - output_dimensions = value.type.tensor_type.shape.dim - if len(output_dimensions) > 1 and output_dimensions[1].dim_value in { - 400, - 1600, - 6400, - }: - output_dimensions[1].ClearField("dim_value") - output_dimensions[1].dim_param = "batched_anchors" - else: # pragma: no cover - internal programming error - raise ValueError(f"Unsupported ONNX model kind: {kind}") + values = ( + list(model.graph.input) + + list(model.graph.output) + + list(model.graph.value_info) + ) + for value in values: + dimensions = value.type.tensor_type.shape.dim + if dimensions and dimensions[0].dim_value == 1: + dimensions[0].ClearField("dim_value") + dimensions[0].dim_param = "batch" return model.SerializeToString() @@ -168,26 +151,24 @@ def __init__(self, config: PrivacyBlurConfig): _require_file(config.face_model_path) self.detector = YOLO(str(config.face_model_path)) - self.yunet_session: ort.InferenceSession | None = None self.sface_session: ort.InferenceSession | None = None - self.reference_feature: torch.Tensor | None = None + self.reference_features: torch.Tensor | None = None + self._identity_generation = 0 if config.enable_identity_exclusion: - _require_file(config.me_image_path) - _require_file(config.face_detection_model_path) + for reference_path in config.identity_image_paths(): + _require_file(reference_path) _require_file(config.face_recognition_model_path) - self.yunet_session = self._create_ort_session( - config.face_detection_model_path, - kind="yunet", - ) self.sface_session = self._create_ort_session( config.face_recognition_model_path, - kind="sface", ) - self.reference_feature = self._load_reference_feature() + self.reference_features = self._load_reference_features(config) self._queue: Queue[_FrameRequest | object] = Queue( maxsize=config.max_pending_frames ) + self._identity_queue: Queue[_IdentityRequest | object] = Queue( + maxsize=config.max_pending_frames * config.max_batch_size + ) self._closed = False self._stats_lock = threading.Lock() self._batches = 0 @@ -201,16 +182,23 @@ def __init__(self, config: PrivacyBlurConfig): self._last_identity_latency_ms = 0.0 self._max_identity_latency_ms = 0.0 self._identity_yolo_landmark_faces = 0 - self._identity_yunet_fallback_faces = 0 + self._identity_missing_landmark_faces = 0 + self._identity_dropped = 0 self._total_identity_latency_ms = 0.0 self._worker = threading.Thread( target=self._worker_loop, name="privacy-blur-gpu-batcher", daemon=True, ) + self._identity_worker = threading.Thread( + target=self._identity_worker_loop, + name="privacy-blur-identity-worker", + daemon=True, + ) self._worker.start() + self._identity_worker.start() - def _create_ort_session(self, path: Path, kind: str) -> ort.InferenceSession: + def _create_ort_session(self, path: Path) -> ort.InferenceSession: available = ort.get_available_providers() if self.cuda_enabled and "CUDAExecutionProvider" not in available: if self.config.require_gpu: @@ -244,9 +232,7 @@ def _create_ort_session(self, path: Path, kind: str) -> ort.InferenceSession: options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL options.intra_op_num_threads = 1 options.log_severity_level = 3 - model_bytes = _dynamic_onnx_model( - path, kind, self.config.identity_detector_size - ) + model_bytes = _dynamic_onnx_model(path) session = ort.InferenceSession( model_bytes, sess_options=options, providers=providers ) @@ -260,33 +246,43 @@ def _create_ort_session(self, path: Path, kind: str) -> ort.InferenceSession: ) return session - def _load_reference_feature(self) -> torch.Tensor: - reference = cv2.imread(str(self.config.me_image_path)) - if reference is None: - raise InvalidReferenceImageError( - f"Failed to read reference image: {self.config.me_image_path}" - ) + def _load_reference_features(self, config: PrivacyBlurConfig) -> torch.Tensor: + reference_faces: list[ + tuple[np.ndarray, np.ndarray, np.ndarray | None] + ] = [] + for reference_path in config.identity_image_paths(): + reference = cv2.imread(str(reference_path)) + if reference is None: + raise InvalidReferenceImageError( + f"Failed to read reference image: {reference_path}" + ) - detector = cv2.FaceDetectorYN.create( - str(self.config.face_detection_model_path), - "", - (self.config.identity_detector_size,) * 2, - self.config.identity_detector_confidence, - 0.3, - 5000, - ) - face = detect_best_yunet_face(detector, reference) - if face is None: - raise ReferenceFaceNotDetectedError( - f"No face found in reference image: {self.config.me_image_path}" + detections = self._predict_yolo([reference])[0] + detections_with_landmarks = [ + detection for detection in detections if detection.landmarks is not None + ] + if not detections_with_landmarks: + raise ReferenceFaceNotDetectedError( + f"No face landmarks found in reference image: {reference_path}" + ) + best = max( + detections_with_landmarks, + key=lambda detection: ( + detection.confidence, + float(detection.xywh[2] * detection.xywh[3]), + ), ) - box = face[:4] - embeddings, valid = self._face_embeddings([(reference, box)]) - if not bool(valid[0]): + reference_faces.append((reference, best.xywh, best.landmarks)) + + if not reference_faces: + raise ReferenceFaceNotDetectedError("No reference images configured") + + embeddings, valid = self._face_embeddings_from_landmarks(reference_faces) + if not bool(valid.all()): raise ReferenceFaceNotDetectedError( - f"YuNet could not align the face in reference image: {self.config.me_image_path}" + "Could not align at least one reference face" ) - return embeddings[0].detach() + return embeddings.detach() def submit( self, engine: FacePrivacyEngine, frame: np.ndarray @@ -365,6 +361,59 @@ def _worker_loop(self) -> None: if not request.future.done(): request.future.set_result(output) + def _identity_worker_loop(self) -> None: + if self.cuda_enabled: + torch.cuda.set_device(self.device) + + while True: + first = self._identity_queue.get() + if first is _STOP: + return + assert isinstance(first, _IdentityRequest) + + requests = [first] + deadline = time.perf_counter() + self.config.batch_wait_ms / 1000.0 + while len(requests) < self.config.max_batch_size: + remaining = deadline - time.perf_counter() + if remaining <= 0: + break + try: + item = self._identity_queue.get(timeout=remaining) + except Empty: + break + if item is _STOP: + self._identity_queue.put_nowait(_STOP) + break + assert isinstance(item, _IdentityRequest) + requests.append(item) + + started = time.perf_counter() + try: + with self._processing_lock: + self._run_identity_batch(requests) + except BaseException: + LOGGER.exception("Identity batch failed") + for request in requests: + request.engine.apply_identity_matches([(request.track, -1.0)]) + continue + + latency_ms = (time.perf_counter() - started) * 1000.0 + landmark_faces = sum(request.landmarks is not None for request in requests) + with self._stats_lock: + self._identity_batches += 1 + self._identity_faces += len(requests) + self._last_identity_batch_size = len(requests) + self._max_identity_batch_size = max( + self._max_identity_batch_size, len(requests) + ) + self._last_identity_latency_ms = latency_ms + self._max_identity_latency_ms = max( + self._max_identity_latency_ms, latency_ms + ) + self._identity_yolo_landmark_faces += landmark_faces + self._identity_missing_landmark_faces += len(requests) - landmark_faces + self._total_identity_latency_ms += latency_ms + def prepare_identity(self, config: PrivacyBlurConfig) -> _PreparedIdentity: """Build a reference embedding while live batches are paused.""" if not config.enable_identity_exclusion: @@ -372,20 +421,10 @@ def prepare_identity(self, config: PrivacyBlurConfig) -> _PreparedIdentity: with self._processing_lock: original_config = self.config - original_yunet = self.yunet_session original_sface = self.sface_session - original_feature = self.reference_feature + original_features = self.reference_features try: self.config = config - if ( - original_yunet is None - or original_config.face_detection_model_path - != config.face_detection_model_path - ): - self.yunet_session = self._create_ort_session( - config.face_detection_model_path, - kind="yunet", - ) if ( original_sface is None or original_config.face_recognition_model_path @@ -393,21 +432,17 @@ def prepare_identity(self, config: PrivacyBlurConfig) -> _PreparedIdentity: ): self.sface_session = self._create_ort_session( config.face_recognition_model_path, - kind="sface", ) - self.reference_feature = self._load_reference_feature() - assert self.yunet_session is not None + self.reference_features = self._load_reference_features(config) assert self.sface_session is not None return _PreparedIdentity( - yunet_session=self.yunet_session, sface_session=self.sface_session, - reference_feature=self.reference_feature, + reference_features=self.reference_features, ) finally: self.config = original_config - self.yunet_session = original_yunet self.sface_session = original_sface - self.reference_feature = original_feature + self.reference_features = original_features def apply_identity( self, @@ -417,14 +452,12 @@ def apply_identity( """Publish prepared identity state between inference batches.""" with self._processing_lock: self.config = config + self._identity_generation += 1 if prepared is None: - self.yunet_session = None - self.sface_session = None - self.reference_feature = None + self.reference_features = None return - self.yunet_session = prepared.yunet_session self.sface_session = prepared.sface_session - self.reference_feature = prepared.reference_feature + self.reference_features = prepared.reference_features @torch.inference_mode() def _run_batch(self, requests: list[_FrameRequest]) -> list[np.ndarray]: @@ -440,52 +473,82 @@ def _run_batch(self, requests: list[_FrameRequest]) -> list[np.ndarray]: ) detections = dict(zip(detect_indices, detected)) - identity_work: list[tuple[FacePrivacyEngine, FaceTrack, np.ndarray]] = [] + identity_work: list[_IdentityRequest] = [] for index, request in enumerate(requests): tracks = request.engine.begin_frame(detections.get(index)) - identity_work.extend( - (request.engine, track, request.frame) for track in tracks - ) + for track in tracks: + identity_work.append( + _IdentityRequest( + engine=request.engine, + track=track, + frame=request.frame, + xywh=track.xywh.copy(), + landmarks=( + track.landmarks.copy() + if track.landmarks is not None + else None + ), + reference_features=request.engine.reference_features, + generation=self._identity_generation, + ) + ) if identity_work: - identity_started = time.perf_counter() - similarities = self._identity_similarities( - [ - (frame, track.xywh, track.landmarks) - for _, track, frame in identity_work - ] - ) - identity_latency_ms = (time.perf_counter() - identity_started) * 1000.0 - grouped: dict[FacePrivacyEngine, list[tuple[FaceTrack, float]]] = ( - defaultdict(list) - ) - for (engine, track, _), similarity in zip(identity_work, similarities): - grouped[engine].append((track, similarity)) - for engine, matches in grouped.items(): - engine.apply_identity_matches(matches) - - landmark_faces = sum( - track.landmarks is not None for _, track, _ in identity_work - ) - with self._stats_lock: - self._identity_batches += 1 - self._identity_faces += len(identity_work) - self._last_identity_batch_size = len(identity_work) - self._max_identity_batch_size = max( - self._max_identity_batch_size, len(identity_work) - ) - self._last_identity_latency_ms = identity_latency_ms - self._max_identity_latency_ms = max( - self._max_identity_latency_ms, identity_latency_ms - ) - self._identity_yolo_landmark_faces += landmark_faces - self._identity_yunet_fallback_faces += ( - len(identity_work) - landmark_faces - ) - self._total_identity_latency_ms += identity_latency_ms + self._submit_identity_work(identity_work) privacy_boxes = [request.engine.finish_frame() for request in requests] return self._blur_frames([request.frame for request in requests], privacy_boxes) + def _submit_identity_work(self, requests: list[_IdentityRequest]) -> None: + dropped: list[_IdentityRequest] = [] + for request in requests: + try: + self._identity_queue.put_nowait(request) + except Full: + dropped.append(request) + + if not dropped: + return + + with self._stats_lock: + self._identity_dropped += len(dropped) + for request in dropped: + request.engine.apply_identity_matches([(request.track, -1.0)]) + + def _run_identity_batch(self, requests: list[_IdentityRequest]) -> None: + active_requests = [ + request + for request in requests + if request.generation == self._identity_generation + ] + stale_requests = [ + request + for request in requests + if request.generation != self._identity_generation + ] + for request in stale_requests: + request.engine.apply_identity_matches([(request.track, -1.0)]) + if not active_requests: + return + + similarities = self._identity_similarities( + [ + ( + request.frame, + request.xywh, + request.landmarks, + request.reference_features, + ) + for request in active_requests + ] + ) + grouped: dict[FacePrivacyEngine, list[tuple[FaceTrack, float]]] = defaultdict( + list + ) + for request, similarity in zip(active_requests, similarities): + grouped[request.engine].append((request.track, similarity)) + for engine, matches in grouped.items(): + engine.apply_identity_matches(matches) + def _predict_yolo( self, frames: list[np.ndarray] ) -> list[list[FaceDetection]]: @@ -585,9 +648,45 @@ def _predict_yolo( def _identity_similarities( self, - faces: list[tuple[np.ndarray, np.ndarray, np.ndarray | None]], + faces: list[ + tuple[ + np.ndarray, + np.ndarray, + np.ndarray | None, + ] + | tuple[ + np.ndarray, + np.ndarray, + np.ndarray | None, + torch.Tensor | None, + ] + ], ) -> list[float]: - if self.reference_feature is None: + if not faces: + return [] + + face_reference_features: list[torch.Tensor | None] = [] + normalized_faces: list[tuple[np.ndarray, np.ndarray, np.ndarray | None]] = [] + for face in faces: + if len(face) == 4: + frame, box, landmarks, reference_features = face + else: + frame, box, landmarks = face + reference_features = getattr(self, "reference_features", None) + legacy_feature = getattr(self, "reference_feature", None) + if reference_features is None and legacy_feature is not None: + reference_features = ( + legacy_feature.unsqueeze(0) + if legacy_feature.ndim == 1 + else legacy_feature + ) + normalized_faces.append((frame, box, landmarks)) + face_reference_features.append(reference_features) + + if not any( + reference_features is not None and reference_features.numel() > 0 + for reference_features in face_reference_features + ): return [-1.0] * len(faces) embeddings = torch.zeros( @@ -595,32 +694,33 @@ def _identity_similarities( ) valid = torch.zeros(len(faces), device=self.device, dtype=torch.bool) landmark_indices = [ - index for index, (_, _, landmarks) in enumerate(faces) if landmarks is not None - ] - fallback_indices = [ - index for index, (_, _, landmarks) in enumerate(faces) if landmarks is None + index + for index, (_, _, landmarks) in enumerate(normalized_faces) + if landmarks is not None ] if landmark_indices: landmark_embeddings, landmark_valid = ( self._face_embeddings_from_landmarks( - [faces[index] for index in landmark_indices] + [normalized_faces[index] for index in landmark_indices] ) ) target = torch.as_tensor(landmark_indices, device=self.device) embeddings[target] = landmark_embeddings valid[target] = landmark_valid - if fallback_indices: - fallback_embeddings, fallback_valid = self._face_embeddings( - [(faces[index][0], faces[index][1]) for index in fallback_indices] - ) - target = torch.as_tensor(fallback_indices, device=self.device) - embeddings[target] = fallback_embeddings - valid[target] = fallback_valid - - scores = embeddings @ self.reference_feature - scores = torch.where(valid, scores, torch.full_like(scores, -1.0)) + scores = torch.full((len(faces),), -1.0, device=self.device, dtype=torch.float32) + for index, reference_features in enumerate(face_reference_features): + if ( + reference_features is None + or reference_features.numel() == 0 + or not bool(valid[index]) + ): + continue + if reference_features.ndim == 1: + reference_features = reference_features.unsqueeze(0) + similarities = embeddings[index : index + 1] @ reference_features.T + scores[index] = similarities.max() return [float(value) for value in scores.detach().float().cpu().tolist()] def _face_embeddings_from_landmarks( @@ -726,137 +826,6 @@ def _face_embeddings_from_landmarks( valid[mapped_indices] = True return embeddings, valid - def _face_embeddings( - self, - faces: list[tuple[np.ndarray, np.ndarray]], - ) -> tuple[torch.Tensor, torch.Tensor]: - size = self.config.identity_detector_size - crops: list[torch.Tensor] = [] - source_indices: list[int] = [] - frame_tensors: dict[int, torch.Tensor] = {} - - for index, (frame, box) in enumerate(faces): - frame_h, frame_w = frame.shape[:2] - x, y, width, height = box[:4].astype(float) - pad_x, pad_y = width * 0.35, height * 0.35 - x1 = max(0, min(frame_w - 1, int(round(x - pad_x)))) - y1 = max(0, min(frame_h - 1, int(round(y - pad_y)))) - x2 = max(0, min(frame_w, int(round(x + width + pad_x)))) - y2 = max(0, min(frame_h, int(round(y + height + pad_y)))) - if x2 <= x1 or y2 <= y1: - continue - - frame_key = id(frame) - frame_tensor = frame_tensors.get(frame_key) - if frame_tensor is None: - frame_tensor = torch.from_numpy(np.ascontiguousarray(frame)).to( - self.device - ) - frame_tensor = frame_tensor.permute(2, 0, 1).float() - frame_tensors[frame_key] = frame_tensor - crop = frame_tensor[:, y1:y2, x1:x2].unsqueeze(0) - crop = F.interpolate( - crop, size=(size, size), mode="bilinear", align_corners=False - ) - crops.append(crop[0]) - source_indices.append(index) - - embeddings = torch.zeros( - (len(faces), 128), device=self.device, dtype=torch.float32 - ) - valid = torch.zeros(len(faces), device=self.device, dtype=torch.bool) - if not crops: - return embeddings, valid - - crop_batch = torch.stack(crops) - stream_context = ( - torch.cuda.stream(self._cuda_stream) - if self._cuda_stream is not None - else torch.no_grad() - ) - with stream_context: - landmarks, landmark_valid = self._yunet_landmarks(crop_batch) - if landmark_valid.any(): - valid_positions = torch.nonzero( - landmark_valid, as_tuple=False - ).flatten() - aligned = self._align_faces( - crop_batch[valid_positions], landmarks[valid_positions] - ) - aligned_rgb = aligned[:, [2, 1, 0], :, :].contiguous() - features = self._run_sface(aligned_rgb) - features = F.normalize(features.float(), dim=1) - mapped_indices = torch.as_tensor(source_indices, device=self.device)[ - valid_positions - ] - embeddings[mapped_indices] = features - valid[mapped_indices] = True - return embeddings, valid - - def _yunet_landmarks( - self, crops: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor]: - if self.yunet_session is None: # pragma: no cover - guarded by configuration - raise RuntimeError("YuNet session is not initialized") - batch_size = crops.shape[0] - outputs = self._run_yunet(crops) - all_landmarks: list[torch.Tensor] = [] - all_ranks: list[torch.Tensor] = [] - size = self.config.identity_detector_size - - for level, stride in enumerate((8, 16, 32)): - rows = size // stride - anchors = rows * rows - cls = outputs[level].reshape(batch_size, anchors, 1).clamp(0, 1) - obj = outputs[level + 3].reshape(batch_size, anchors, 1).clamp(0, 1) - bbox = outputs[level + 6].reshape(batch_size, anchors, 4) - keypoints = outputs[level + 9].reshape(batch_size, anchors, 5, 2) - - score = torch.sqrt(cls * obj).squeeze(-1) - grid_cache = getattr(self, "_yunet_grid_cache", None) - if grid_cache is None: - grid_cache = {} - self._yunet_grid_cache = grid_cache - grid = grid_cache.get((size, stride)) - if grid is None: - grid_y, grid_x = torch.meshgrid( - torch.arange(rows, device=self.device, dtype=torch.float32), - torch.arange(rows, device=self.device, dtype=torch.float32), - indexing="ij", - ) - grid = torch.stack((grid_x.flatten(), grid_y.flatten()), dim=-1) - grid_cache[(size, stride)] = grid - width = torch.exp(bbox[..., 2].clamp(max=10)) * stride - height = torch.exp(bbox[..., 3].clamp(max=10)) * stride - rank = width * height * score - rank = torch.where( - score >= self.config.identity_detector_confidence, - rank, - torch.full_like(rank, -1.0), - ) - decoded = (keypoints + grid.view(1, anchors, 1, 2)) * stride - all_landmarks.append(decoded) - all_ranks.append(rank) - - landmarks = torch.cat(all_landmarks, dim=1) - ranks = torch.cat(all_ranks, dim=1) - best_rank, best_index = ranks.max(dim=1) - batch_indices = torch.arange(batch_size, device=self.device) - return landmarks[batch_indices, best_index], best_rank >= 0 - - def _run_yunet(self, inputs: torch.Tensor) -> list[torch.Tensor]: - assert self.yunet_session is not None - batch_size = inputs.shape[0] - size = self.config.identity_detector_size - output_shapes: list[tuple[int, ...]] = [] - for channels in (1, 1, 4, 10): - for stride in (8, 16, 32): - anchors = (size // stride) ** 2 - output_shapes.append((1, batch_size * anchors, channels)) - return self._run_ort( - self.yunet_session, inputs.float().contiguous(), output_shapes - ) - def _run_sface(self, inputs: torch.Tensor) -> torch.Tensor: assert self.sface_session is not None return self._run_ort( @@ -1115,6 +1084,7 @@ def stats(self) -> dict[str, Any]: else [] ), "queue_depth": self._queue.qsize(), + "identity_queue_depth": self._identity_queue.qsize(), "batches": self._batches, "frames": self._frames, "last_batch_size": self._last_batch_size, @@ -1131,7 +1101,9 @@ def stats(self) -> dict[str, Any]: self._max_identity_latency_ms, 2 ), "identity_yolo_landmark_faces": self._identity_yolo_landmark_faces, - "identity_yunet_fallback_faces": self._identity_yunet_fallback_faces, + "identity_missing_landmark_faces": self._identity_missing_landmark_faces, + "identity_yunet_fallback_faces": 0, + "identity_dropped": self._identity_dropped, "average_identity_batch_size": round(average_identity_batch, 2), "average_identity_latency_ms": round(average_identity_latency, 2), } @@ -1144,7 +1116,12 @@ def close(self) -> None: self._queue.put_nowait(_STOP) except Full: self._queue.put(_STOP) + try: + self._identity_queue.put_nowait(_STOP) + except Full: + self._identity_queue.put(_STOP) self._worker.join(timeout=10) + self._identity_worker.join(timeout=10) _RUNTIMES: dict[PrivacyBlurConfig, SharedGpuRuntime] = {} diff --git a/tests/conftest.py b/tests/conftest.py index 8e11228..bb57b33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,11 @@ import numpy as np import pytest -from app.services.ai.identity import REFERENCE_FACE_DIR, REFERENCE_FACE_PATH +from app.services.ai.identity import ( + LEGACY_REFERENCE_FACE_PATH, + REFERENCE_FACE_DIR, + REFERENCE_FACE_ROOT, +) @pytest.fixture @@ -113,8 +117,20 @@ def _create_test_jpeg() -> bytes: @pytest.fixture(autouse=True) def cleanup_reference_face(): """각 테스트 후 data/ 디렉토리를 정리한다.""" + _cleanup_reference_face_files() yield - REFERENCE_FACE_PATH.unlink(missing_ok=True) + _cleanup_reference_face_files() + + +def _cleanup_reference_face_files() -> None: + LEGACY_REFERENCE_FACE_PATH.unlink(missing_ok=True) + if REFERENCE_FACE_ROOT.exists(): + for path in sorted(REFERENCE_FACE_ROOT.rglob("*"), reverse=True): + if path.is_file(): + path.unlink(missing_ok=True) + elif path.is_dir(): + path.rmdir() + REFERENCE_FACE_ROOT.rmdir() if REFERENCE_FACE_DIR.exists() and not any(REFERENCE_FACE_DIR.iterdir()): REFERENCE_FACE_DIR.rmdir() diff --git a/tests/test_api_reference_face.py b/tests/test_api_reference_face.py index f5bef9d..70dfc28 100644 --- a/tests/test_api_reference_face.py +++ b/tests/test_api_reference_face.py @@ -6,7 +6,7 @@ import pytest from httpx import ASGITransport, AsyncClient -from app.services.ai.identity import REFERENCE_FACE_PATH +from app.services.ai.identity import get_reference_face_paths from tests.conftest import _create_test_jpeg pytestmark = pytest.mark.asyncio @@ -64,7 +64,85 @@ async def test_post_reference_face_saves_file(client): files={"image": ("face.jpg", BytesIO(image_data), "image/jpeg")}, ) - assert REFERENCE_FACE_PATH.exists() + paths = get_reference_face_paths() + assert len(paths) == 1 + assert paths[0].read_bytes() == image_data + + +async def test_post_reference_face_single_image_replaces_existing(client): + """기존 단일 image 업로드 클라이언트는 재업로드 시 기존 사진을 교체한다.""" + first_image = _create_test_jpeg() + second_image = _create_test_jpeg() + b"second" + + with patch("app.services.ai.identity._validate_image"): + await client.post( + "/reference-face", + files={"image": ("face-1.jpg", BytesIO(first_image), "image/jpeg")}, + ) + response = await client.post( + "/reference-face", + files={"image": ("face-2.jpg", BytesIO(second_image), "image/jpeg")}, + ) + + assert response.status_code == 201 + paths = get_reference_face_paths() + assert len(paths) == 1 + assert paths[0].read_bytes() == second_image + + +async def test_post_reference_face_accepts_multiple_images(client): + """여러 이미지 업로드 시 같은 클라이언트 기준 얼굴 목록에 추가된다.""" + image_data = _create_test_jpeg() + + with patch("app.services.ai.identity._validate_image"): + response = await client.post( + "/reference-face?client_id=client-a", + files=[ + ("images", ("face-1.jpg", BytesIO(image_data), "image/jpeg")), + ("images", ("face-2.jpg", BytesIO(image_data), "image/jpeg")), + ], + ) + + assert response.status_code == 201 + body = response.json() + assert body["client_id"] == "client-a" + assert body["count"] == 2 + assert len(body["faces"]) == 2 + assert len(get_reference_face_paths("client-a")) == 2 + + +async def test_reference_face_is_isolated_by_client_id(client): + """클라이언트별 등록 사진은 조회와 삭제에서 분리된다.""" + image_data = _create_test_jpeg() + + with patch("app.services.ai.identity._validate_image"): + await client.post( + "/reference-face?client_id=client-a", + files={"image": ("face.jpg", BytesIO(image_data), "image/jpeg")}, + ) + await client.post( + "/reference-face?client_id=client-b", + files={"image": ("face.jpg", BytesIO(image_data), "image/jpeg")}, + ) + + response_a = await client.get("/reference-face?client_id=client-a") + response_b = await client.get("/reference-face?client_id=client-b") + + assert response_a.json()["registered"] is True + assert response_a.json()["client_id"] == "client-a" + assert response_a.json()["count"] == 1 + assert response_b.json()["registered"] is True + assert response_b.json()["client_id"] == "client-b" + assert response_b.json()["count"] == 1 + + await client.delete("/reference-face?client_id=client-a") + + assert (await client.get("/reference-face?client_id=client-a")).json()[ + "registered" + ] is False + assert (await client.get("/reference-face?client_id=client-b")).json()[ + "registered" + ] is True async def test_post_reference_face_unsupported_type(client): @@ -170,7 +248,7 @@ async def test_delete_reference_face_removes_file(client): await client.delete("/reference-face") - assert not REFERENCE_FACE_PATH.exists() + assert get_reference_face_paths() == [] async def test_delete_reference_face_status_after_delete(client): @@ -203,6 +281,60 @@ async def test_delete_reference_face_when_not_registered(client): assert response.status_code == 204 +async def test_delete_reference_face_by_id_removes_only_one_image(client): + """face_id 지정 DELETE는 해당 이미지만 삭제한다.""" + image_data = _create_test_jpeg() + with patch("app.services.ai.identity._validate_image"): + create_response = await client.post( + "/reference-face?client_id=client-a", + files=[ + ("images", ("face-1.jpg", BytesIO(image_data), "image/jpeg")), + ("images", ("face-2.jpg", BytesIO(image_data), "image/jpeg")), + ], + ) + + faces = create_response.json()["faces"] + removed_face_id = faces[0]["face_id"] + kept_face_id = faces[1]["face_id"] + + response = await client.delete( + f"/reference-face/{removed_face_id}?client_id=client-a" + ) + + assert response.status_code == 204 + status = (await client.get("/reference-face?client_id=client-a")).json() + assert status["registered"] is True + assert status["count"] == 1 + assert status["faces"][0]["face_id"] == kept_face_id + assert len(get_reference_face_paths("client-a")) == 1 + + +async def test_delete_reference_face_by_id_returns_404_for_missing_id(client): + """없는 face_id를 삭제하려 하면 404를 반환한다.""" + response = await client.delete("/reference-face/missing") + + assert response.status_code == 404 + assert response.json()["error"]["details"]["reason"] == "reference_face_not_found" + + +async def test_delete_reference_face_by_id_is_scoped_by_client_id(client): + """다른 클라이언트의 face_id는 삭제할 수 없다.""" + image_data = _create_test_jpeg() + with patch("app.services.ai.identity._validate_image"): + create_response = await client.post( + "/reference-face?client_id=client-a", + files={"image": ("face.jpg", BytesIO(image_data), "image/jpeg")}, + ) + face_id = create_response.json()["faces"][0]["face_id"] + + response = await client.delete(f"/reference-face/{face_id}?client_id=client-b") + + assert response.status_code == 404 + assert (await client.get("/reference-face?client_id=client-a")).json()[ + "registered" + ] is True + + # --------------------------------------------------------------------------- # POST /reference-face — file size limit # --------------------------------------------------------------------------- diff --git a/tests/test_face_detection.py b/tests/test_face_detection.py deleted file mode 100644 index 71297da..0000000 --- a/tests/test_face_detection.py +++ /dev/null @@ -1,65 +0,0 @@ -"""YuNet 기준 얼굴 감지 전처리 테스트.""" - -import numpy as np - -from privacy_blur.face_detection import detect_best_yunet_face - - -class FakeDetector: - def __init__(self, faces_by_size): - self.faces_by_size = faces_by_size - self.input_size = None - self.detected_shapes = [] - - def setInputSize(self, input_size): - self.input_size = input_size - - def detect(self, image): - self.detected_shapes.append(image.shape[:2]) - return 1, self.faces_by_size.get(self.input_size) - - -def _face(score=0.9): - return np.array( - [[ - 64, 96, 128, 160, - 96, 128, 160, 128, 128, 160, - 104, 208, 152, 208, - score, - ]], - dtype=np.float32, - ) - - -def test_high_resolution_reference_is_downscaled_and_mapped_back(): - image = np.zeros((2000, 1000, 3), dtype=np.uint8) - detector = FakeDetector({(320, 640): _face()}) - - face = detect_best_yunet_face(detector, image) - - assert detector.detected_shapes == [(640, 320)] - assert face is not None - np.testing.assert_allclose(face[:-1], _face()[0, :-1] / 0.32) - assert face[-1] == np.float32(0.9) - - -def test_reference_detection_retries_at_smaller_scale(): - image = np.zeros((1200, 800, 3), dtype=np.uint8) - detector = FakeDetector({(213, 320): _face()}) - - face = detect_best_yunet_face(detector, image) - - assert detector.detected_shapes == [(640, 427), (320, 213)] - assert face is not None - expected = _face()[0].copy() - expected[[0, 2, 4, 6, 8, 10, 12]] /= 213 / 800 - expected[[1, 3, 5, 7, 9, 11, 13]] /= 320 / 1200 - np.testing.assert_allclose(face[:-1], expected[:-1]) - - -def test_reference_detection_returns_none_after_all_scales_fail(): - image = np.zeros((1200, 800, 3), dtype=np.uint8) - detector = FakeDetector({}) - - assert detect_best_yunet_face(detector, image) is None - assert detector.detected_shapes == [(640, 427), (320, 213)] diff --git a/tests/test_identity_performance.py b/tests/test_identity_performance.py index be98939..436e699 100644 --- a/tests/test_identity_performance.py +++ b/tests/test_identity_performance.py @@ -34,7 +34,7 @@ def _engine(**overrides) -> FacePrivacyEngine: return FacePrivacyEngine(config=config, runtime=MagicMock()) -def test_deferred_new_track_remains_due_until_compared() -> None: +def test_pending_new_track_is_not_queued_again_until_compared() -> None: engine = _engine() first = engine.begin_frame([_detection()]) @@ -42,8 +42,9 @@ def test_deferred_new_track_remains_due_until_compared() -> None: second = engine.begin_frame([_detection()]) assert len(first) == 1 - assert second[0] is first[0] - assert second[0].last_checked_frame == -1 + assert first[0].identity_pending is True + assert second == [] + assert first[0].last_checked_frame == -1 def test_non_reference_track_uses_longer_recheck_interval() -> None: @@ -52,6 +53,7 @@ def test_non_reference_track_uses_longer_recheck_interval() -> None: engine.apply_identity_matches([(track, 0.1)]) engine.finish_frame() + assert track.identity_pending is False for _ in range(9): assert engine.begin_frame([_detection()]) == [] engine.finish_frame() @@ -66,6 +68,7 @@ def test_reference_track_keeps_short_recheck_interval() -> None: engine.apply_identity_matches([(track, 0.9)]) engine.finish_frame() + assert track.identity_pending is False for _ in range(2): assert engine.begin_frame([_detection()]) == [] engine.finish_frame() @@ -74,6 +77,17 @@ def test_reference_track_keeps_short_recheck_interval() -> None: assert engine.begin_frame([_detection()]) == [track] +def test_failed_identity_check_does_not_delay_next_attempt() -> None: + engine = _engine() + track = engine.begin_frame([_detection()])[0] + engine.apply_identity_matches([(track, -1.0)]) + engine.finish_frame() + + assert track.last_checked_frame == -1 + assert track.identity_pending is False + assert engine.begin_frame([_detection()]) == [track] + + def test_vectorized_track_assignment_matches_scalar_greedy_result() -> None: engine = _engine() rng = np.random.default_rng(42) @@ -117,7 +131,7 @@ def test_vectorized_track_assignment_matches_scalar_greedy_result() -> None: ] -def test_all_detector_landmarks_use_one_sface_batch_without_yunet() -> None: +def test_all_detector_landmarks_use_one_sface_batch() -> None: runtime = SharedGpuRuntime.__new__(SharedGpuRuntime) runtime.device = torch.device("cpu") runtime._cuda_stream = None @@ -129,9 +143,6 @@ def test_all_detector_landmarks_use_one_sface_batch_without_yunet() -> None: (inputs.shape[0], 128), dtype=torch.float32 ) ) - runtime._face_embeddings = MagicMock( - side_effect=AssertionError("YuNet fallback should not run") - ) frame = np.zeros((120, 120, 3), dtype=np.uint8) detection = _detection() @@ -142,4 +153,20 @@ def test_all_detector_landmarks_use_one_sface_batch_without_yunet() -> None: assert min(scores) > 0.99 runtime._run_sface.assert_called_once() assert runtime._run_sface.call_args.args[0].shape == (32, 3, 112, 112) - runtime._face_embeddings.assert_not_called() + + +def test_missing_landmarks_are_not_identity_matched() -> None: + runtime = SharedGpuRuntime.__new__(SharedGpuRuntime) + runtime.device = torch.device("cpu") + runtime.reference_features = torch.nn.functional.normalize( + torch.ones((1, 128), dtype=torch.float32), dim=1 + ) + runtime._run_sface = MagicMock() + frame = np.zeros((120, 120, 3), dtype=np.uint8) + + scores = runtime._identity_similarities( + [(frame, np.array([20, 20, 80, 80], dtype=np.float32), None)] + ) + + assert scores == [-1.0] + runtime._run_sface.assert_not_called() diff --git a/tests/test_on_track.py b/tests/test_on_track.py index 07244de..f92ec14 100644 --- a/tests/test_on_track.py +++ b/tests/test_on_track.py @@ -115,7 +115,7 @@ async def test_wrap_called_on_video_track(session_manager, mock_peer_connection) with patch("app.services.sessions.manager._wrap_with_ai", return_value=track) as mock_wrap: on_track(track) - mock_wrap.assert_called_once_with(track, session.session_id) + mock_wrap.assert_called_once_with(track, session.session_id, client_id=None) async def test_video_track_attached_to_prepared_sender( @@ -163,7 +163,10 @@ async def test_previous_track_stopped_on_replace(session_manager, mock_peer_conn wrap_results = iter([wrapped1, wrapped2]) - with patch("app.services.sessions.manager._wrap_with_ai", side_effect=lambda t, s: next(wrap_results)): + with patch( + "app.services.sessions.manager._wrap_with_ai", + side_effect=lambda t, s, **_: next(wrap_results), + ): on_track(track1) on_track(track2) @@ -239,7 +242,10 @@ async def test_on_track_ended_ignores_other_track(session_manager, mock_peer_con track1 = _make_track("video") track2 = _make_track("video") - with patch("app.services.sessions.manager._wrap_with_ai", side_effect=lambda t, s: t): + with patch( + "app.services.sessions.manager._wrap_with_ai", + side_effect=lambda t, s, **_: t, + ): on_track(track1) on_track(track2) @@ -324,7 +330,10 @@ async def test_previous_raw_stopped_on_replace_when_ai_disabled( track1 = _make_track("video") track2 = _make_track("video") - with patch("app.services.sessions.manager._wrap_with_ai", side_effect=lambda t, s: t): + with patch( + "app.services.sessions.manager._wrap_with_ai", + side_effect=lambda t, s, **_: t, + ): on_track(track1) on_track(track2) diff --git a/tests/test_performance_runtime.py b/tests/test_performance_runtime.py index 8021498..cf633f7 100644 --- a/tests/test_performance_runtime.py +++ b/tests/test_performance_runtime.py @@ -17,15 +17,13 @@ def make_test_config(**overrides) -> PrivacyBlurConfig: defaults = { - "face_model_path": ROOT / "models/yolov12n-face.pt", + "face_model_path": ROOT / "models/yolo11s-pose_widerface.pt", "me_image_path": ROOT / "me.jpeg", - "face_detection_model_path": ROOT / "models/face_detection_yunet_2023mar.onnx", "face_recognition_model_path": ROOT / "models/face_recognition_sface_2021dec.onnx", "device": "cpu", "require_gpu": False, "face_imgsz": 320, - "identity_detector_size": 320, "max_batch_size": 4, "batch_wait_ms": 10, "blur_kernel": 21, @@ -35,6 +33,19 @@ def make_test_config(**overrides) -> PrivacyBlurConfig: return PrivacyBlurConfig(**defaults) +async def wait_for_identity_matches(filters, timeout: float = 2.0) -> None: + deadline = asyncio.get_running_loop().time() + timeout + while True: + if all( + item.stats()["tracks"] and item.stats()["tracks"][0]["is_me"] + for item in filters + ): + return + if asyncio.get_running_loop().time() >= deadline: + return + await asyncio.sleep(0.01) + + class ConfigTests(unittest.TestCase): def test_production_mode_rejects_cpu(self) -> None: with self.assertRaises(PrivacyBlurNotReadyError): @@ -70,6 +81,7 @@ async def test_filters_share_models_and_frames_are_dynamically_batched( self.assertEqual(stats["frames"], 4) self.assertEqual(stats["batches"], 1) self.assertEqual(stats["last_batch_size"], 4) + await wait_for_identity_matches(filters) self.assertTrue(all(item.stats()["tracks"][0]["is_me"] for item in filters)) async def test_non_reference_faces_receive_a_masked_gpu_pipeline_result( @@ -106,6 +118,7 @@ async def test_reference_identity_can_be_hot_swapped_on_shared_runtime( output = await privacy_filter.apply_async(frame) self.assertEqual(output.shape, frame.shape) + await wait_for_identity_matches([privacy_filter]) self.assertTrue(privacy_filter.stats()["tracks"][0]["is_me"]) disabled_config = identity_config.with_overrides( diff --git a/tests/test_privacy_blur.py b/tests/test_privacy_blur.py index a633134..9ac4810 100644 --- a/tests/test_privacy_blur.py +++ b/tests/test_privacy_blur.py @@ -15,8 +15,7 @@ def _build_config(**overrides): from privacy_blur import PrivacyBlurConfig defaults = { - "face_model_path": ROOT / "models/yolov12n-face.pt", - "face_detection_model_path": ROOT / "models/face_detection_yunet_2023mar.onnx", + "face_model_path": ROOT / "models/yolo11s-pose_widerface.pt", "face_recognition_model_path": ROOT / "models/face_recognition_sface_2021dec.onnx", "enable_identity_exclusion": False, diff --git a/tests/test_reference_face_service.py b/tests/test_reference_face_service.py index 7903217..0396a7f 100644 --- a/tests/test_reference_face_service.py +++ b/tests/test_reference_face_service.py @@ -4,13 +4,18 @@ import pytest -from app.core.exceptions import BadRequestError +from app.core.exceptions import BadRequestError, NotFoundError from app.services.ai.identity import ( REFERENCE_FACE_DIR, - REFERENCE_FACE_PATH, + DEFAULT_REFERENCE_CLIENT_ID, + LEGACY_REFERENCE_FACE_PATH, + ReferenceFaceUpload, delete_reference_face, + delete_reference_face_by_id, + get_reference_face_paths, get_reference_face_info, register_reference_face, + register_reference_faces, ) from tests.conftest import _create_test_jpeg @@ -23,6 +28,8 @@ def running_runtime(mock_privacy_blur_module): candidate_config = MagicMock(name="candidate_config") final_config = MagicMock(name="final_config") disabled_config = MagicMock(name="disabled_config") + disabled_config.enable_identity_exclusion = False + mock_privacy_blur_module.PrivacyBlurConfig.return_value = disabled_config base_config.with_overrides.side_effect = lambda **kwargs: ( disabled_config if kwargs == {"enable_identity_exclusion": False} @@ -32,6 +39,7 @@ def running_runtime(mock_privacy_blur_module): prepared = MagicMock(name="prepared_identity") mock_privacy_blur_module.runtime.prepare_identity.return_value = prepared running_filter = MagicMock(name="running_filter") + running_filter.config.identity_client_id = DEFAULT_REFERENCE_CLIENT_ID filter_module._config = base_config filter_module._config_initialized = True @@ -53,14 +61,17 @@ def test_register_prepares_once_and_updates_all_streams(running_runtime): assert result.registered is True assert result.source == "api" - assert REFERENCE_FACE_PATH.read_bytes() == b"image bytes" + paths = get_reference_face_paths() + assert len(paths) == 1 + assert paths[0].read_bytes() == b"image bytes" running_runtime["runtime"].prepare_identity.assert_called_once() running_runtime["runtime"].apply_identity.assert_called_once_with( running_runtime["final"], running_runtime["prepared"], ) running_runtime["filter"].replace_config.assert_called_once_with( - running_runtime["final"] + running_runtime["final"], + running_runtime["prepared"].reference_features, ) @@ -68,22 +79,46 @@ def test_register_uses_candidate_then_publishes_final_path(running_runtime): register_reference_face(b"image bytes") candidate_kwargs = running_runtime["base"].with_overrides.call_args.kwargs - candidate_path = candidate_kwargs["me_image_path"] + [candidate_path] = candidate_kwargs["me_image_paths"] assert candidate_path.name.endswith(".tmp") + assert candidate_kwargs["me_image_path"] == candidate_path assert candidate_kwargs["enable_identity_exclusion"] is True running_runtime["candidate"].with_overrides.assert_called_once() - final_path = running_runtime["candidate"].with_overrides.call_args.kwargs[ - "me_image_path" - ] - assert final_path.name == REFERENCE_FACE_PATH.name + final_kwargs = running_runtime["candidate"].with_overrides.call_args.kwargs + [final_path] = final_kwargs["me_image_paths"] + assert final_kwargs["me_image_path"] == final_path + assert final_path.suffix == ".jpg" assert final_path.is_absolute() +def test_single_register_replaces_existing_reference(running_runtime): + register_reference_face(b"first image") + register_reference_face(b"second image") + + paths = get_reference_face_paths() + assert len(paths) == 1 + assert paths[0].read_bytes() == b"second image" + + +def test_register_faces_appends_new_references(running_runtime): + register_reference_faces([ReferenceFaceUpload(image_data=b"first image")]) + register_reference_faces([ReferenceFaceUpload(image_data=b"second image")]) + + paths = get_reference_face_paths() + assert len(paths) == 2 + assert sorted(path.read_bytes() for path in paths) == [ + b"first image", + b"second image", + ] + + def test_invalid_replacement_preserves_existing_reference(running_runtime): from privacy_blur.errors import ReferenceFaceNotDetectedError REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"old valid image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"old valid image") running_runtime["runtime"].prepare_identity.side_effect = ( ReferenceFaceNotDetectedError("No face found") ) @@ -91,14 +126,16 @@ def test_invalid_replacement_preserves_existing_reference(running_runtime): with pytest.raises(BadRequestError, match="얼굴을 감지할 수 없습니다"): register_reference_face(b"invalid replacement") - assert REFERENCE_FACE_PATH.read_bytes() == b"old valid image" + assert existing.read_bytes() == b"old valid image" running_runtime["runtime"].apply_identity.assert_not_called() running_runtime["filter"].replace_config.assert_not_called() def test_runtime_failure_is_not_reported_as_face_not_detected(running_runtime): REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"old valid image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"old valid image") running_runtime["runtime"].prepare_identity.side_effect = RuntimeError( "CUDA execution failed" ) @@ -106,7 +143,7 @@ def test_runtime_failure_is_not_reported_as_face_not_detected(running_runtime): with pytest.raises(RuntimeError, match="CUDA execution failed"): register_reference_face(b"replacement image") - assert REFERENCE_FACE_PATH.read_bytes() == b"old valid image" + assert existing.read_bytes() == b"old valid image" running_runtime["runtime"].apply_identity.assert_not_called() @@ -117,42 +154,91 @@ def test_register_without_runtime_validates_and_saves_file(): validate.assert_called_once_with(image_data) assert result.registered is True - assert REFERENCE_FACE_PATH.read_bytes() == image_data + paths = get_reference_face_paths() + assert len(paths) == 1 + assert paths[0].read_bytes() == image_data + + +def test_register_multiple_without_runtime_validates_and_saves_files(): + image_data = _create_test_jpeg() + with patch("app.services.ai.identity._validate_image") as validate: + result = register_reference_faces( + [ + ReferenceFaceUpload(image_data=image_data), + ReferenceFaceUpload(image_data=image_data, extension=".png"), + ], + client_id="client-a", + ) + + assert validate.call_count == 2 + assert result.registered is True + assert result.client_id == "client-a" + assert result.count == 2 + assert len(get_reference_face_paths("client-a")) == 2 def test_invalid_image_without_runtime_preserves_existing_file(): REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"old valid image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"old valid image") with pytest.raises(BadRequestError, match="유효하지 않은 이미지"): register_reference_face(b"not an image") - assert REFERENCE_FACE_PATH.read_bytes() == b"old valid image" + assert existing.read_bytes() == b"old valid image" def test_delete_disables_runtime_identity_and_resets_streams(running_runtime): REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"image") delete_reference_face() - assert not REFERENCE_FACE_PATH.exists() + assert not existing.exists() running_runtime["runtime"].apply_identity.assert_called_once_with( running_runtime["disabled"], None, ) running_runtime["filter"].replace_config.assert_called_once_with( - running_runtime["disabled"] + running_runtime["disabled"], + None, ) def test_delete_without_runtime_removes_file(): REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"image") delete_reference_face() - assert not REFERENCE_FACE_PATH.exists() + assert not existing.exists() + + +def test_delete_by_id_removes_only_matching_reference_without_runtime(): + client_dir = REFERENCE_FACE_DIR / "reference_faces" / "default" + client_dir.mkdir(parents=True, exist_ok=True) + first = client_dir / "first.jpg" + second = client_dir / "second.jpg" + first.write_bytes(b"first image") + second.write_bytes(b"second image") + + info = delete_reference_face_by_id("first") + + assert info.registered is True + assert info.count == 1 + assert not first.exists() + assert second.exists() + assert get_reference_face_paths()[0].stem == "second" + + +def test_delete_by_id_raises_not_found_for_missing_reference(): + with pytest.raises(NotFoundError): + delete_reference_face_by_id("missing") def test_get_info_prefers_api_reference(monkeypatch): @@ -161,13 +247,16 @@ def test_get_info_prefers_api_reference(monkeypatch): get_settings.cache_clear() REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) - REFERENCE_FACE_PATH.write_bytes(b"image") + existing = REFERENCE_FACE_DIR / "reference_faces" / "default" / "existing.jpg" + existing.parent.mkdir(parents=True, exist_ok=True) + existing.write_bytes(b"image") info = get_reference_face_info() assert info.registered is True assert info.source == "api" assert info.registered_at is not None + assert info.count == 1 def test_get_info_reports_environment_reference(monkeypatch): @@ -187,3 +276,18 @@ def test_get_info_reports_not_registered(): assert info.registered is False assert info.source is None + + +def test_get_info_reads_legacy_default_reference(monkeypatch): + monkeypatch.setenv("AI_PRIVACY_ME_IMAGE_PATH", "/path/to/me.jpg") + from app.core.config import get_settings + + get_settings.cache_clear() + REFERENCE_FACE_DIR.mkdir(parents=True, exist_ok=True) + LEGACY_REFERENCE_FACE_PATH.write_bytes(b"legacy image") + + info = get_reference_face_info() + + assert info.registered is True + assert info.source == "api" + assert info.count == 1 diff --git a/tests/test_runtime_onnx.py b/tests/test_runtime_onnx.py index 23eae10..a5f66bb 100644 --- a/tests/test_runtime_onnx.py +++ b/tests/test_runtime_onnx.py @@ -8,15 +8,12 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1] -def test_yunet_output_anchor_dimensions_remain_dynamic(): +def test_sface_batch_dimension_remains_dynamic(): model = _dynamic_onnx_model( - PROJECT_ROOT / "models" / "face_detection_yunet_2023mar.onnx", - "yunet", - 320, + PROJECT_ROOT / "models" / "face_recognition_sface_2021dec.onnx", ) session = ort.InferenceSession(model, providers=["CPUExecutionProvider"]) - assert all( - output.shape[1] == "batched_anchors" for output in session.get_outputs() - ) + assert session.get_inputs()[0].shape[0] == "batch" + assert session.get_outputs()[0].shape[0] == "batch" diff --git a/tests/test_static_client.py b/tests/test_static_client.py index 1d2edfe..aa5f7f9 100644 --- a/tests/test_static_client.py +++ b/tests/test_static_client.py @@ -28,7 +28,8 @@ def test_client_assets_served(): assert script_response.status_code == 200 assert "RTCPeerConnection" in script_response.text - assert 'apiFetch("/reference-face"' in script_response.text + assert "referenceFaceApiPath()" in script_response.text + assert "CLIENT_ID_STORAGE_KEY" in script_response.text assert "body instanceof FormData" in script_response.text assert 'location.protocol === "file:"' in script_response.text start_web_rtc = script_response.text.split( diff --git a/tests/test_track.py b/tests/test_track.py index 9d2bcac..676867d 100644 --- a/tests/test_track.py +++ b/tests/test_track.py @@ -1,6 +1,8 @@ """ProtectedVideoTrack 단위 테스트.""" +import asyncio from fractions import Fraction +from types import SimpleNamespace from unittest.mock import AsyncMock, patch import numpy as np @@ -44,6 +46,33 @@ async def test_recv_calls_source_then_filter( assert result is processed_frame +async def test_recv_drops_stale_source_frames_before_filter(mock_privacy_filter): + """source 내부 큐에 쌓인 과거 프레임은 버리고 최신 프레임만 AI 처리한다.""" + from app.services.ai.track import ProtectedVideoTrack + + stale = _video_frame(30) + newer = _video_frame(90) + latest = _video_frame(150) + source_queue = asyncio.Queue() + source_queue.put_nowait(newer) + source_queue.put_nowait(latest) + source = SimpleNamespace( + _queue=source_queue, + recv=AsyncMock(return_value=stale), + stop=AsyncMock(), + ) + track = ProtectedVideoTrack( + source=source, + privacy_filter=mock_privacy_filter, + session_id="test-session-id", + ) + + await track.recv() + + mock_privacy_filter.apply_async.assert_called_once_with(latest) + assert source_queue.empty() + + async def test_recv_source_exception_propagates( protected_track, mock_source_track, mock_privacy_filter ): diff --git a/tests/test_track_tee.py b/tests/test_track_tee.py index 27f4847..9feae83 100644 --- a/tests/test_track_tee.py +++ b/tests/test_track_tee.py @@ -6,7 +6,7 @@ from aiortc.mediastreams import MediaStreamError from app.services.sessions.manager import SessionManager -from app.services.webrtc.track_tee import VideoTrackBroadcaster +from app.services.webrtc.track_tee import TeeVideoTrack, VideoTrackBroadcaster @pytest.fixture @@ -74,6 +74,18 @@ async def recv(): ) +async def test_tee_video_track_returns_latest_queued_frame(): + old_frame = object() + latest_frame = object() + queue = asyncio.Queue() + queue.put_nowait(old_frame) + queue.put_nowait(latest_frame) + track = TeeVideoTrack(queue) + + assert await track.recv() is latest_frame + assert queue.empty() + + async def test_source_failure_marks_active_stream_error(manager): frame = MagicMock() image = MagicMock()