401 lines
14 KiB
Python
401 lines
14 KiB
Python
import cv2
|
||
import mediapipe as mp
|
||
import time
|
||
import numpy as np
|
||
import threading
|
||
import queue
|
||
import multiprocessing as mp_proc
|
||
from multiprocessing import shared_memory
|
||
from collections import deque
|
||
from geometry_utils import (
|
||
calculate_ear,
|
||
calculate_mar_simple,
|
||
calculate_iris_pos,
|
||
estimate_head_pose,
|
||
LEFT_EYE,
|
||
RIGHT_EYE,
|
||
LEFT_EYE_GAZE_IDXS,
|
||
RIGHT_EYE_GAZE_IDXS,
|
||
)
|
||
# from face_library import FaceLibrary
|
||
|
||
# try:
|
||
# from new_emotion_test import analyze_emotion_with_hsemotion
|
||
|
||
# HAS_EMOTION_MODULE = True
|
||
# except ImportError:
|
||
# print("⚠️ 未找到 new_emotion_test.py，情绪功能将不可用")
|
||
# HAS_EMOTION_MODULE = False
|
||
|
||
|
||
class MonitorSystem:
|
||
def __init__(self, face_db):
|
||
# 初始化 MediaPipe
|
||
self.mp_face_mesh = mp.solutions.face_mesh
|
||
self.face_mesh = self.mp_face_mesh.FaceMesh(
|
||
max_num_faces=1,
|
||
refine_landmarks=True,
|
||
min_detection_confidence=0.5,
|
||
min_tracking_confidence=0.5,
|
||
)
|
||
|
||
# 初始化人脸底库
|
||
# self.face_lib = FaceLibrary(face_db)
|
||
|
||
# --- 时间控制 ---
|
||
self.last_identity_check_time = 0
|
||
self.IDENTITY_CHECK_INTERVAL = 2.0
|
||
|
||
self.last_emotion_check_time = 0
|
||
self.EMOTION_CHECK_INTERVAL = 3.0
|
||
|
||
# --- 历史数据 ---
|
||
self.HISTORY_LEN = 5
|
||
self.ear_history = deque(maxlen=self.HISTORY_LEN)
|
||
self.mar_history = deque(maxlen=self.HISTORY_LEN)
|
||
self.iris_ratio_history = [
|
||
deque(maxlen=self.HISTORY_LEN),
|
||
deque(maxlen=self.HISTORY_LEN),
|
||
]
|
||
|
||
# 缓存上一次的检测结果
|
||
self.cached_emotion = {"label": "detecting...", "va": (0.0, 0.0)}
|
||
|
||
self.current_user = None
|
||
self.current_emotion = "Neutral"
|
||
|
||
self.frame_shape = (720, 1280, 3)
|
||
frame_size = int(np.prod(self.frame_shape))
|
||
|
||
# 必须先解除可能存在的残留 (Windows上有时不需要，但保持好习惯)
|
||
# 最好是随机生成一个名字，确保每次运行都是新的
|
||
import secrets
|
||
auth_key = secrets.token_hex(4)
|
||
shm_unique_name = f"monitor_shm_{auth_key}"
|
||
|
||
try:
|
||
self.shm = shared_memory.SharedMemory(create=True, size=frame_size, name=shm_unique_name)
|
||
except FileExistsError:
|
||
# 如果真的点背碰上了，就 connect 这一块
|
||
self.shm = shared_memory.SharedMemory(name=shm_unique_name)
|
||
|
||
print(f"[Main] 共享内存已创建: {self.shm.name} (Size: {frame_size} bytes)")
|
||
|
||
# 本地 numpy 包装器
|
||
self.shared_frame_array = np.ndarray(
|
||
self.frame_shape, dtype=np.uint8, buffer=self.shm.buf
|
||
)
|
||
# 初始化为全黑，避免噪音
|
||
self.shared_frame_array.fill(0)
|
||
|
||
# 跨进程队列
|
||
self.task_queue = mp_proc.Queue(maxsize=2)
|
||
self.result_queue = mp_proc.Queue(maxsize=2) # 1就够了，最新的覆盖
|
||
|
||
# 3. 启动进程
|
||
# Windows下传参，只传名字字符串是安全的
|
||
self.worker_proc = mp_proc.Process(
|
||
target=background_worker_process,
|
||
args=(
|
||
self.shm.name,
|
||
self.frame_shape,
|
||
self.task_queue,
|
||
self.result_queue,
|
||
face_db,
|
||
),
|
||
)
|
||
self.worker_proc.daemon = True
|
||
self.worker_proc.start()
|
||
|
||
def _get_smoothed_value(self, history, current_val):
|
||
"""内部函数：计算滑动平均值"""
|
||
history.append(current_val)
|
||
if len(history) == 0:
|
||
return current_val
|
||
return sum(history) / len(history)
|
||
|
||
def process_frame(self, frame):
|
||
"""
|
||
输入 BGR 图像，返回分析结果字典
|
||
"""
|
||
# 强制检查分辨率，如果不匹配则 Resize (对应 __init__ 中硬编码的 1280x720)
|
||
# 这一步至关重要，否则后台进程读到的全是黑屏
|
||
target_h, target_w = self.frame_shape[:2]
|
||
if frame.shape[:2] != (target_h, target_w):
|
||
frame = cv2.resize(frame, (target_w, target_h))
|
||
|
||
h, w = frame.shape[:2]
|
||
# 现在肯定匹配了，放心写入
|
||
try:
|
||
self.shared_frame_array[:] = frame[:]
|
||
except Exception:
|
||
# 极端情况：数组形状不匹配 (比如通道数变了)
|
||
pass
|
||
|
||
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
|
||
|
||
results = self.face_mesh.process(rgb_frame)
|
||
|
||
analysis_data = {
|
||
"has_face": False,
|
||
"ear": 0.0,
|
||
"mar": 0.0,
|
||
"iris_ratio": (0.5, 0.5), # 0最左/上，1最右/下
|
||
"pose": (0, 0, 0),
|
||
"identity": self.current_user,
|
||
"emotion_label": self.cached_emotion["label"],
|
||
"emotion_va": self.cached_emotion["va"],
|
||
"landmark": (0, w, h, 0),
|
||
"frame": frame,
|
||
}
|
||
|
||
if not results.multi_face_landmarks:
|
||
self.ear_history.clear()
|
||
self.mar_history.clear()
|
||
self.iris_ratio_history[0].clear()
|
||
self.iris_ratio_history[1].clear()
|
||
return analysis_data
|
||
|
||
analysis_data["has_face"] = True
|
||
landmarks = results.multi_face_landmarks[0].landmark
|
||
|
||
# 计算 EAR
|
||
left_ear = calculate_ear([landmarks[i] for i in LEFT_EYE], w, h)
|
||
right_ear = calculate_ear([landmarks[i] for i in RIGHT_EYE], w, h)
|
||
raw_ear = (left_ear + right_ear) / 2.0
|
||
|
||
# 计算 MAR
|
||
top = np.array([landmarks[13].x * w, landmarks[13].y * h])
|
||
bottom = np.array([landmarks[14].x * w, landmarks[14].y * h])
|
||
left = np.array([landmarks[78].x * w, landmarks[78].y * h])
|
||
right = np.array([landmarks[308].x * w, landmarks[308].y * h])
|
||
raw_mar = calculate_mar_simple(top, bottom, left, right)
|
||
|
||
# 计算虹膜位置
|
||
left_iris_ratio = calculate_iris_pos(landmarks, LEFT_EYE_GAZE_IDXS, w, h)
|
||
right_iris_ratio = calculate_iris_pos(landmarks, RIGHT_EYE_GAZE_IDXS, w, h)
|
||
raw_iris_ratio = (
|
||
(left_iris_ratio[0] + right_iris_ratio[0]) / 2.0,
|
||
(left_iris_ratio[1] + right_iris_ratio[1]) / 2.0,
|
||
)
|
||
|
||
# --- 使用 History 进行数据平滑 ---
|
||
smoothed_ear = self._get_smoothed_value(self.ear_history, raw_ear)
|
||
smoothed_mar = self._get_smoothed_value(self.mar_history, raw_mar)
|
||
smoothed_iris_ratio = (
|
||
(self._get_smoothed_value(self.iris_ratio_history[0], raw_iris_ratio[0])),
|
||
(self._get_smoothed_value(self.iris_ratio_history[1], raw_iris_ratio[1])),
|
||
)
|
||
|
||
# 计算头部姿态
|
||
pitch, yaw, roll = estimate_head_pose(landmarks, w, h)
|
||
|
||
analysis_data.update(
|
||
{
|
||
"ear": round(smoothed_ear, 4),
|
||
"mar": round(smoothed_mar, 4),
|
||
"iris_ratio": (
|
||
round(smoothed_iris_ratio[0], 4),
|
||
round(smoothed_iris_ratio[1], 4),
|
||
),
|
||
"pose": (int(pitch), int(yaw), int(roll)),
|
||
}
|
||
)
|
||
|
||
xs = [l.x for l in landmarks]
|
||
ys = [l.y for l in landmarks]
|
||
# 计算人脸框
|
||
face_loc = (
|
||
int(min(ys) * h - 0.1 * h),
|
||
int(max(xs) * w + 0.1 * w),
|
||
int(max(ys) * h + 0.1 * h),
|
||
int(min(xs) * w - 0.1 * w),
|
||
)
|
||
pad = 30
|
||
face_loc = (
|
||
max(0, face_loc[0] - pad),
|
||
min(w, face_loc[1] + pad),
|
||
min(h, face_loc[2] + pad),
|
||
max(0, face_loc[3] - pad),
|
||
)
|
||
analysis_data["landmark"] = face_loc
|
||
|
||
# --- ROI处理(对比选择在哪里实现) ---
|
||
top = face_loc[0]
|
||
right = face_loc[1]
|
||
bottom = face_loc[2]
|
||
left = face_loc[3]
|
||
scale_factor = 10
|
||
small_bg = cv2.resize(
|
||
frame,
|
||
(w // scale_factor, h // scale_factor),
|
||
interpolation=cv2.INTER_LINEAR,
|
||
)
|
||
# 使用 INTER_NEAREST 马赛克效果
|
||
# 使用 INTER_LINEAR 毛玻璃模糊效果
|
||
blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
|
||
|
||
face_roi = frame[top:bottom, left:right]
|
||
|
||
blurred_frame[top:bottom, left:right] = face_roi
|
||
analysis_data["frame"] = blurred_frame
|
||
|
||
now = time.time()
|
||
# --- 身份识别 ---
|
||
if now - self.last_identity_check_time > self.IDENTITY_CHECK_INTERVAL:
|
||
sface_loc = (
|
||
int(min(ys) * h),
|
||
int(max(xs) * w),
|
||
int(max(ys) * h),
|
||
int(min(xs) * w),
|
||
)
|
||
spad = 20
|
||
sface_loc = (
|
||
max(0, sface_loc[0] - spad),
|
||
min(w, sface_loc[1] + spad),
|
||
min(h, sface_loc[2] + spad),
|
||
max(0, sface_loc[3] - spad),
|
||
)
|
||
|
||
if self.task_queue.full():
|
||
self.task_queue.get()
|
||
self.task_queue.put((sface_loc, 0))
|
||
|
||
self.last_identity_check_time = now
|
||
|
||
# --- 情绪识别 ---
|
||
if (
|
||
now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL
|
||
):
|
||
|
||
# 计算裁剪坐标
|
||
x_min = int(min(xs) * w)
|
||
x_max = int(max(xs) * w)
|
||
y_min = int(min(ys) * h)
|
||
y_max = int(max(ys) * h)
|
||
|
||
pad_x = int((x_max - x_min) * 0.1)
|
||
pad_y = int((y_max - y_min) * 0.1)
|
||
|
||
x_min = max(0, x_min - pad_x)
|
||
x_max = min(w, x_max + pad_x)
|
||
y_min = max(0, y_min - pad_y)
|
||
y_max = min(h, y_max + pad_y)
|
||
|
||
face_loc = (y_min, x_max, y_max, x_min)
|
||
|
||
if self.task_queue.full():
|
||
self.task_queue.get()
|
||
self.task_queue.put((face_loc, 1))
|
||
|
||
self.last_emotion_check_time = now
|
||
|
||
while not self.result_queue.empty():
|
||
type_, data = self.result_queue.get()
|
||
if type_ == "identity":
|
||
self.current_user = data
|
||
elif type_ == "emotion":
|
||
self.cached_emotion["label"] = data.get("emotion", "unknown")
|
||
self.cached_emotion["va"] = data.get("vaVal", (0.0, 0.0))
|
||
|
||
analysis_data["identity"] = self.current_user
|
||
analysis_data["emotion_label"] = self.cached_emotion["label"]
|
||
analysis_data["emotion_va"] = self.cached_emotion["va"]
|
||
|
||
return analysis_data
|
||
|
||
# def _id_emo_loop(self):
|
||
# while True:
|
||
# try:
|
||
# frame, face_loc, task_type = self.task_queue.get()
|
||
# if task_type == 0:
|
||
# match_result = self.face_lib.identify(frame, face_location=face_loc)
|
||
# if match_result:
|
||
# self.current_user = match_result["info"]
|
||
# elif task_type == 1 and HAS_EMOTION_MODULE:
|
||
# face_crop = frame[
|
||
# face_loc[0] : face_loc[2], face_loc[3] : face_loc[1]
|
||
# ]
|
||
|
||
# if face_crop.size > 0:
|
||
# try:
|
||
# emo_results = analyze_emotion_with_hsemotion(face_crop)
|
||
|
||
# if emo_results:
|
||
# top_res = emo_results[0]
|
||
# self.cached_emotion["label"] = top_res.get(
|
||
# "emotion", "unknown"
|
||
# )
|
||
# self.cached_emotion["va"] = top_res.get(
|
||
# "vaVal", (0.0, 0.0)
|
||
# )
|
||
|
||
# except Exception as e:
|
||
# print(f"情绪分析出错: {e}")
|
||
# except Exception as e:
|
||
# print(f"线程处理出错: {e}")
|
||
|
||
|
||
def background_worker_process(
|
||
shm_name, # 共享内存的名字
|
||
frame_shape, # 图像大小 (h, w, 3)
|
||
task_queue, # 任务队列 (主 -> 从)
|
||
result_queue, # 结果队列 (从 -> 主)
|
||
face_db_data, # 把人脸库数据传过去初始化
|
||
):
|
||
existing_shm = shared_memory.SharedMemory(name=shm_name)
|
||
# 创建 numpy 数组视图，无需复制数据
|
||
shared_frame = np.ndarray(frame_shape, dtype=np.uint8, buffer=existing_shm.buf)
|
||
|
||
print("[Worker] 正在加载模型...")
|
||
from face_library import FaceLibrary
|
||
|
||
face_lib = FaceLibrary(face_db_data)
|
||
|
||
try:
|
||
from new_emotion_test import analyze_emotion_with_hsemotion
|
||
|
||
has_emo = True
|
||
except:
|
||
has_emo = False
|
||
print("[Worker] 模型加载完毕")
|
||
|
||
while True:
|
||
try:
|
||
# 阻塞等待任务
|
||
# task_info = (task_type, face_loc)
|
||
face_loc, task_type = task_queue.get()
|
||
|
||
# 注意：这里读取的是共享内存里的图，不需要传图！
|
||
# 切片操作也是零拷贝
|
||
# 为了安全，这里 copy 一份出来处理，避免主进程修改
|
||
# 但实际上如果主进程只写新帧，这里读旧帧也问题不大
|
||
# 为了绝对安全和解耦，我们假定主进程已经写入了对应的帧
|
||
|
||
# (实战技巧：通常我们会用一个信号量或多块共享内存来实现乒乓缓存)
|
||
# 简化版：我们直接从 shared_frame 读。
|
||
# 由于主进程跑得快，可能SharedMemory里已经是下一帧了。
|
||
# 但对于识别身份来说，差一两帧根本没区别！这才是优化的精髓。
|
||
|
||
current_frame_view = shared_frame.copy() # .copy() 如果你怕读写冲突
|
||
|
||
if task_type == 0: # Identity
|
||
# RGB转换
|
||
rgb = cv2.cvtColor(current_frame_view, cv2.COLOR_BGR2RGB)
|
||
res = face_lib.identify(rgb, face_location=face_loc)
|
||
if res:
|
||
result_queue.put(("identity", res["info"]))
|
||
|
||
elif task_type == 1 and has_emo: # Emotion
|
||
# BGR 直接切
|
||
roi = current_frame_view[
|
||
face_loc[0] : face_loc[2], face_loc[3] : face_loc[1]
|
||
]
|
||
if roi.size > 0:
|
||
emo_res = analyze_emotion_with_hsemotion(roi)
|
||
if emo_res:
|
||
result_queue.put(("emotion", emo_res[0]))
|
||
|
||
except Exception as e:
|
||
print(f"[Worker Error] {e}")
|