diff --git a/reproject/HeartRateMonitor.py b/reproject/HeartRateMonitor.py new file mode 100644 index 0000000..ab30e39 --- /dev/null +++ b/reproject/HeartRateMonitor.py @@ -0,0 +1,139 @@ +import numpy as np +import collections +from scipy import signal + +class HeartRateMonitor: + def __init__(self, fps=30, window_size=300): + self.fps = fps + self.buffer_size = window_size + + # 存储 RGB 三个通道的均值 + self.times = np.zeros(window_size) + self.r_buffer = collections.deque(maxlen=window_size) + self.g_buffer = collections.deque(maxlen=window_size) + self.b_buffer = collections.deque(maxlen=window_size) + + # 滤波器状态 + self.bp_b, self.bp_a = self._create_bandpass_filter(0.75, 2.5, fps) # 45-150 BPM + + # 平滑结果用的 + self.bpm_history = collections.deque(maxlen=10) + + def _create_bandpass_filter(self, lowcut, highcut, fs, order=5): + """创建巴特沃斯带通滤波器""" + nyq = 0.5 * fs + low = lowcut / nyq + high = highcut / nyq + b, a = signal.butter(order, [low, high], btype='band') + return b, a + + def _pos_algorithm(self, r, g, b): + """ + POS (Plane-Orthogonal-to-Skin) 算法 + 比单纯的绿色通道法强在一个地方：抗运动干扰 + """ + # 1. 归一化 (除以均值) + # 加上 1e-6 防止除零 + r_n = r / (np.mean(r) + 1e-6) + g_n = g / (np.mean(g) + 1e-6) + b_n = b / (np.mean(b) + 1e-6) + + # 2. 投影到色度平面 (Matplotlib 里的经典公式) + # S1 = G - B + # S2 = G + B - 2R + s1 = g_n - b_n + s2 = g_n + b_n - 2 * r_n + + # 3. Alpha 微调 (Alpha Tuning) + # 这一步是为了消除镜面反射带来的运动噪声 + alpha = np.std(s1) / (np.std(s2) + 1e-6) + + # 4. 融合信号 + h = s1 + alpha * s2 + return h + + def process_frame(self, frame, face_loc): + """ + 输入: 原始无损 frame (BGR), 人脸框 (top, right, bottom, left) + 输出: BPM 数值 或 None (数据不够时) + """ + top, right, bottom, left = face_loc + + # --- 1. ROI 提取与保护 --- + h_img, w_img = frame.shape[:2] + + # 缩小 ROI 范围：只取脸中心 50% 区域 (避开背景和边缘) + h_box = bottom - top + w_box = right - left + + # 修正 ROI 坐标 + roi_top = int(max(0, top + h_box * 0.3)) + roi_bottom = int(min(h_img, bottom - h_box * 0.3)) + roi_left = int(max(0, left + w_box * 0.3)) + roi_right = int(min(w_img, right - w_box * 0.3)) + + roi = frame[roi_top:roi_bottom, roi_left:roi_right] + + if roi.size == 0: + return None + + # --- 2. 提取 RGB 均值 --- + # OpenCV 是 BGR + b_mean = np.mean(roi[:, :, 0]) + g_mean = np.mean(roi[:, :, 1]) + r_mean = np.mean(roi[:, :, 2]) + + self.r_buffer.append(r_mean) + self.g_buffer.append(g_mean) + self.b_buffer.append(b_mean) + + # 数据不够，返回 None + if len(self.r_buffer) < self.buffer_size: + progress = int(len(self.r_buffer) / self.buffer_size * 100) + return None # 或者返回 progress 表示进度 + + # --- 3. 信号处理 (核心升级部分) --- + r = np.array(self.r_buffer) + g = np.array(self.g_buffer) + b = np.array(self.b_buffer) + + # A. 使用 POS 算法融合三通道 (抗干扰) + pulse_signal = self._pos_algorithm(r, g, b) + + # B. 消除直流分量 (Detrending) + # 这一步去掉了光线缓慢变化的干扰 + pulse_signal = signal.detrend(pulse_signal) + + # C. 带通滤波 (Bandpass Filter) + # 只保留 0.75Hz - 2.5Hz 的信号 + pulse_signal = signal.filtfilt(self.bp_b, self.bp_a, pulse_signal) + + # --- 4. 频域分析 (FFT) --- + # 加汉宁窗 (减少频谱泄露) + window = np.hanning(len(pulse_signal)) + pulse_signal_windowed = pulse_signal * window + + # FFT + fft_res = np.fft.rfft(pulse_signal_windowed) + freqs = np.fft.rfftfreq(len(pulse_signal), 1.0/self.fps) + mag = np.abs(fft_res) + + # D. 寻找峰值 + # 限制频率范围 (45 BPM - 180 BPM) + interest_idx = np.where((freqs >= 0.75) & (freqs <= 3.0)) + valid_freqs = freqs[interest_idx] + valid_mags = mag[interest_idx] + + if len(valid_mags) == 0: + return None + + max_idx = np.argmax(valid_mags) + peak_freq = valid_freqs[max_idx] + bpm = peak_freq * 60.0 + + # --- 5. 结果平滑 --- + # 防止数字乱跳 + self.bpm_history.append(bpm) + avg_bpm = np.mean(self.bpm_history) + + return int(avg_bpm) \ No newline at end of file diff --git a/reproject/analyzer.py b/reproject/analyzer.py index d0dc6c2..4766e05 100644 --- a/reproject/analyzer.py +++ b/reproject/analyzer.py @@ -3,17 +3,28 @@ import mediapipe as mp import time import numpy as np from collections import deque -from geometry_utils import calculate_ear, calculate_mar_simple, estimate_head_pose, LEFT_EYE, RIGHT_EYE +from geometry_utils import ( + calculate_ear, + calculate_mar_simple, + calculate_iris_pos, + estimate_head_pose, + LEFT_EYE, + RIGHT_EYE, + LEFT_EYE_GAZE_IDXS, + RIGHT_EYE_GAZE_IDXS, +) from face_library import FaceLibrary try: from new_emotion_test import analyze_emotion_with_hsemotion + HAS_EMOTION_MODULE = True except ImportError: print("⚠️ 未找到 new_emotion_test.py，情绪功能将不可用") HAS_EMOTION_MODULE = False + class MonitorSystem: def __init__(self, face_db): # 初始化 MediaPipe @@ -22,32 +33,33 @@ class MonitorSystem: max_num_faces=1, refine_landmarks=True, min_detection_confidence=0.5, - min_tracking_confidence=0.5 + min_tracking_confidence=0.5, ) - + # 初始化人脸底库 self.face_lib = FaceLibrary(face_db) - + # 状态变量 self.current_user = None - + # --- 时间控制 --- self.last_identity_check_time = 0 - self.IDENTITY_CHECK_INTERVAL = 2.0 - + self.IDENTITY_CHECK_INTERVAL = 2.0 + self.last_emotion_check_time = 0 - self.EMOTION_CHECK_INTERVAL = 3.0 + self.EMOTION_CHECK_INTERVAL = 3.0 # --- 历史数据 --- self.HISTORY_LEN = 5 self.ear_history = deque(maxlen=self.HISTORY_LEN) self.mar_history = deque(maxlen=self.HISTORY_LEN) - - # 缓存上一次的检测结果 - self.cached_emotion = { - "label": "detecting...", - "va": (0.0, 0.0) - } + self.iris_ratio_history = [ + deque(maxlen=self.HISTORY_LEN), + deque(maxlen=self.HISTORY_LEN), + ] + + # 缓存上一次的检测结果 + self.cached_emotion = {"label": "detecting...", "va": (0.0, 0.0)} def _get_smoothed_value(self, history, current_val): """内部函数：计算滑动平均值""" @@ -62,32 +74,37 @@ class MonitorSystem: """ h, w = frame.shape[:2] rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) - + results = self.face_mesh.process(rgb_frame) - + analysis_data = { "has_face": False, - "ear": 0.0, + "ear": 0.0, "mar": 0.0, + "iris_ratio": (0.5, 0.5), # 0最左/上，1最右/下 "pose": (0, 0, 0), "identity": self.current_user, "emotion_label": self.cached_emotion["label"], - "emotion_va": self.cached_emotion["va"] + "emotion_va": self.cached_emotion["va"], + "landmark": (0, w, h, 0), + "frame": frame, } if not results.multi_face_landmarks: self.ear_history.clear() self.mar_history.clear() + self.iris_ratio_history[0].clear() + self.iris_ratio_history[1].clear() return analysis_data analysis_data["has_face"] = True landmarks = results.multi_face_landmarks[0].landmark - - # 计算 EAR + + # 计算 EAR left_ear = calculate_ear([landmarks[i] for i in LEFT_EYE], w, h) right_ear = calculate_ear([landmarks[i] for i in RIGHT_EYE], w, h) raw_ear = (left_ear + right_ear) / 2.0 - + # 计算 MAR top = np.array([landmarks[13].x * w, landmarks[13].y * h]) bottom = np.array([landmarks[14].x * w, landmarks[14].y * h]) @@ -95,79 +112,136 @@ class MonitorSystem: right = np.array([landmarks[308].x * w, landmarks[308].y * h]) raw_mar = calculate_mar_simple(top, bottom, left, right) + # 计算虹膜位置 + left_iris_ratio = calculate_iris_pos(landmarks, LEFT_EYE_GAZE_IDXS, w, h) + right_iris_ratio = calculate_iris_pos(landmarks, RIGHT_EYE_GAZE_IDXS, w, h) + raw_iris_ratio = ( + (left_iris_ratio[0] + right_iris_ratio[0]) / 2.0, + (left_iris_ratio[1] + right_iris_ratio[1]) / 2.0, + ) + # --- 使用 History 进行数据平滑 --- smoothed_ear = self._get_smoothed_value(self.ear_history, raw_ear) smoothed_mar = self._get_smoothed_value(self.mar_history, raw_mar) + smoothed_iris_ratio = ( + (self._get_smoothed_value(self.iris_ratio_history[0], raw_iris_ratio[0])), + (self._get_smoothed_value(self.iris_ratio_history[1], raw_iris_ratio[1])), + ) # 计算头部姿态 pitch, yaw, roll = estimate_head_pose(landmarks, w, h) - analysis_data.update({ - "ear": round(smoothed_ear, 4), - "mar": round(smoothed_mar, 4), - "pose": (int(pitch), int(yaw), int(roll)) - }) + analysis_data.update( + { + "ear": round(smoothed_ear, 4), + "mar": round(smoothed_mar, 4), + "iris_ratio": ( + round(smoothed_iris_ratio[0], 4), + round(smoothed_iris_ratio[1], 4), + ), + "pose": (int(pitch), int(yaw), int(roll)), + } + ) + + xs = [l.x for l in landmarks] + ys = [l.y for l in landmarks] + # 计算人脸框 + face_loc = ( + int(min(ys) * h - 0.1 * h), + int(max(xs) * w + 0.1 * w), + int(max(ys) * h + 0.1 * h), + int(min(xs) * w - 0.1 * w), + ) + pad = 30 + face_loc = ( + max(0, face_loc[0] - pad), + min(w, face_loc[1] + pad), + min(h, face_loc[2] + pad), + max(0, face_loc[3] - pad), + ) + analysis_data["landmark"] = face_loc + + # --- ROI处理(对比选择在哪里实现) --- + top = face_loc[0] + right = face_loc[1] + bottom = face_loc[2] + left = face_loc[3] + scale_factor = 10 + small_bg = cv2.resize( + frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR + ) + # 使用 INTER_NEAREST 马赛克效果 + # 使用 INTER_LINEAR 毛玻璃模糊效果 + blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR) + + face_roi = frame[top:bottom, left:right] + + blurred_frame[top:bottom, left:right] = face_roi + analysis_data["frame"] = blurred_frame now = time.time() - # --- 身份识别 --- if now - self.last_identity_check_time > self.IDENTITY_CHECK_INTERVAL: - xs = [l.x for l in landmarks] - ys = [l.y for l in landmarks] - # 计算人脸框 - face_loc = ( - int(min(ys) * h), int(max(xs) * w), - int(max(ys) * h), int(min(xs) * w) - ) - pad = 20 - face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad), - min(h, face_loc[2]+pad), max(0, face_loc[3]-pad)) + # xs = [l.x for l in landmarks] + # ys = [l.y for l in landmarks] + # # 计算人脸框 + # face_loc = ( + # int(min(ys) * h), int(max(xs) * w), + # int(max(ys) * h), int(min(xs) * w) + # ) + # pad = 20 + # face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad), + # min(h, face_loc[2]+pad), max(0, face_loc[3]-pad)) match_result = self.face_lib.identify(rgb_frame, face_location=face_loc) if match_result: self.current_user = match_result["info"] self.last_identity_check_time = now - + analysis_data["identity"] = self.current_user # --- 情绪识别 --- - if HAS_EMOTION_MODULE and (now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL): + if HAS_EMOTION_MODULE and ( + now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL + ): if results.multi_face_landmarks: landmarks = results.multi_face_landmarks[0].landmark xs = [l.x for l in landmarks] ys = [l.y for l in landmarks] - + # 计算裁剪坐标 x_min = int(min(xs) * w) x_max = int(max(xs) * w) y_min = int(min(ys) * h) y_max = int(max(ys) * h) - + pad_x = int((x_max - x_min) * 0.2) pad_y = int((y_max - y_min) * 0.2) - + x_min = max(0, x_min - pad_x) x_max = min(w, x_max + pad_x) y_min = max(0, y_min - pad_y) y_max = min(h, y_max + pad_y) - + face_crop = frame[y_min:y_max, x_min:x_max] - + if face_crop.size > 0: try: emo_results = analyze_emotion_with_hsemotion(face_crop) - + if emo_results: top_res = emo_results[0] - self.cached_emotion["label"] = top_res.get("emotion", "unknown") + self.cached_emotion["label"] = top_res.get( + "emotion", "unknown" + ) self.cached_emotion["va"] = top_res.get("vaVal", (0.0, 0.0)) - + except Exception as e: print(f"情绪分析出错: {e}") - + self.last_emotion_check_time = now analysis_data["emotion_label"] = self.cached_emotion["label"] analysis_data["emotion_va"] = self.cached_emotion["va"] - return analysis_data \ No newline at end of file + return analysis_data diff --git a/reproject/geometry_utils.py b/reproject/geometry_utils.py index 403ad17..b03bc7c 100644 --- a/reproject/geometry_utils.py +++ b/reproject/geometry_utils.py @@ -5,71 +5,115 @@ import cv2 LEFT_EYE = [33, 160, 158, 133, 153, 144] # 右眼 RIGHT_EYE = [362, 385, 387, 263, 373, 380] +# 左眼虹膜关键点索引 +LEFT_EYE_GAZE_IDXS = [33, 133, 159, 145, 468] +# 右眼虹膜关键点索引 +RIGHT_EYE_GAZE_IDXS = [263, 362, 386, 374, 473] # 嘴唇 (内圈) LIPS = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, 311, 312, 13] + def _euclidean_distance(point1, point2): return np.linalg.norm(point1 - point2) + def calculate_ear(landmarks, width, height): """计算眼睛纵横比 EAR""" # 坐标转换 points = np.array([(p.x * width, p.y * height) for p in landmarks]) - + # 垂直距离 v1 = _euclidean_distance(points[1], points[5]) v2 = _euclidean_distance(points[2], points[4]) # 水平距离 h = _euclidean_distance(points[0], points[3]) - + ear = (v1 + v2) / (2.0 * h) return ear + +def calculate_iris_pos(landmarks, indices, width, height): + p_left = np.array([landmarks[indices[0]].x * width, landmarks[indices[0]].y * height]) + p_right = np.array([landmarks[indices[1]].x * width, landmarks[indices[1]].y * height]) + p_top = np.array([landmarks[indices[2]].x * width, landmarks[indices[2]].y * height]) + p_bottom = np.array([landmarks[indices[3]].x * width, landmarks[indices[3]].y * height]) + p_iris = np.array([landmarks[indices[4]].x * width, landmarks[indices[4]].y * height]) + + # 修改为欧几里得距离计算 + eye_width = _euclidean_distance(p_left, p_right) + iris_left = _euclidean_distance(p_iris, p_left) + if eye_width < 1e-3: + raw_x = 0.5 + else : + raw_x = iris_left / eye_width + eye_height = _euclidean_distance(p_bottom, p_top) + iris_top = _euclidean_distance(p_iris, p_top) + if eye_height < 1e-3: + raw_y = 0.5 + else: + raw_y = iris_top / eye_height + + # x_min = 0.4 + # x_max = 0.6 + # ratio_x = abs(raw_x - x_min) / (x_max - x_min) + # y_min = 0.2 + # y_max = 0.8 + # ratio_y = abs(raw_y - y_min) / (y_max - y_min) + ratio_x = raw_x + ratio_y = raw_y + + return max(0.0, min(1.0, ratio_x)), max(0.0, min(1.0, ratio_y)) + + def calculate_mar(landmarks, width, height): """计算嘴巴纵横比 MAR""" points = np.array([(p.x * width, p.y * height) for p in landmarks]) - pass + pass + def calculate_mar_simple(top, bottom, left, right): h = _euclidean_distance(top, bottom) w = _euclidean_distance(left, right) return h / w + # geometry_utils.py 中的 estimate_head_pose 函数替换为以下内容 + def estimate_head_pose(landmarks, width, height): """ 计算头部姿态 (Pitch, Yaw, Roll) 返回单位：角度 (Degree) """ # 3D 模型点 (标准人脸模型) - model_points = np.array([ - (0.0, 0.0, 0.0), # Nose tip - (0.0, -330.0, -65.0), # Chin - (-225.0, 170.0, -135.0), # Left eye left corner - (225.0, 170.0, -135.0), # Right eye right corner - (-150.0, -150.0, -125.0), # Left Mouth corner - (150.0, -150.0, -125.0) # Right mouth corner - ]) + model_points = np.array( + [ + (0.0, 0.0, 0.0), # Nose tip + (0.0, -330.0, -65.0), # Chin + (-225.0, 170.0, -135.0), # Left eye left corner + (225.0, 170.0, -135.0), # Right eye right corner + (-150.0, -150.0, -125.0), # Left Mouth corner + (150.0, -150.0, -125.0), # Right mouth corner + ] + ) # MediaPipe 对应的关键点索引 idx_list = [1, 152, 33, 263, 61, 291] - + image_points = [] for idx in idx_list: p = landmarks[idx] image_points.append((p.x * width, p.y * height)) - + image_points = np.array(image_points, dtype="double") focal_length = width center = (width / 2, height / 2) camera_matrix = np.array( - [[focal_length, 0, center[0]], - [0, focal_length, center[1]], - [0, 0, 1]], dtype="double" + [[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]], + dtype="double", ) - dist_coeffs = np.zeros((4, 1)) + dist_coeffs = np.zeros((4, 1)) # 求解PnP success, rotation_vector, translation_vector = cv2.solvePnP( @@ -80,15 +124,21 @@ def estimate_head_pose(landmarks, width, height): angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat) pitch = angles[0] - yaw = angles[1] - roll = angles[2] + yaw = angles[1] + roll = angles[2] - if pitch < -180: pitch += 360 - if pitch > 180: pitch -= 360 + if pitch < -180: + pitch += 360 + if pitch > 180: + pitch -= 360 pitch = 180 - pitch if pitch > 0 else -pitch - 180 - if yaw < -180: yaw += 360 - if yaw > 180: yaw -= 360 - if roll < -180: roll += 360 - if roll > 180: roll -= 360 + if yaw < -180: + yaw += 360 + if yaw > 180: + yaw -= 360 + if roll < -180: + roll += 360 + if roll > 180: + roll -= 360 - return pitch, yaw, roll \ No newline at end of file + return pitch, yaw, roll diff --git a/reproject/main.py b/reproject/main.py index 99aa199..5ad2399 100644 --- a/reproject/main.py +++ b/reproject/main.py @@ -1,3 +1,4 @@ +from calendar import c import cv2 import threading import time @@ -5,12 +6,16 @@ import queue import socket import json import urllib.request -import struct +import struct +import numpy as np +import mediapipe as mp from analyzer import MonitorSystem +from webrtc_server import WebRTCServer +from HeartRateMonitor import HeartRateMonitor -SERVER_HOST = '10.128.50.6' -SERVER_PORT = 65432 -API_URL = "http://10.128.50.6:5000/api/states" +SERVER_HOST = "10.128.50.6" +SERVER_PORT = 65432 +API_URL = "http://10.128.50.6:5000/api/states" CAMERA_ID = "23373333" BASIC_FACE_DB = { @@ -18,32 +23,89 @@ BASIC_FACE_DB = { "Yaoyu": {"name": "Yaoyu Zhang", "age": 20, "image-path": "yaoyu.jpg"}, } +mp_face_mesh_main = mp.solutions.face_mesh +face_mesh_main = mp_face_mesh_main.FaceMesh( + max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=0.5, + min_tracking_confidence=0.5, +) -frame_queue = queue.Queue(maxsize=2) -video_queue = queue.Queue(maxsize=1) +def apply_soft_roi(frame): + h, w = frame.shape[:2] + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + black_bg = np.zeros((h, w, 3), dtype=np.uint8) -data_queue = queue.Queue(maxsize=10) + results = face_mesh_main.process(rgb_frame) + if not results.multi_face_landmarks: + return frame + landmarks = results.multi_face_landmarks[0].landmark + xs = [l.x for l in landmarks] + ys = [l.y for l in landmarks] + # 计算人脸框 + face_loc = ( + int(min(ys) * h - 0.1 * h), + int(max(xs) * w + 0.1 * w), + int(max(ys) * h + 0.1 * h), + int(min(xs) * w - 0.1 * w), + ) + pad = 30 + face_loc = ( + max(0, face_loc[0] - pad), + min(w, face_loc[1] + pad), + min(h, face_loc[2] + pad), + max(0, face_loc[3] - pad), + ) + top = face_loc[0] + right = face_loc[1] + bottom = face_loc[2] + left = face_loc[3] + scale_factor = 10 + small_bg = cv2.resize( + frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR + ) + # 使用 INTER_NEAREST 马赛克效果 + # 使用 INTER_LINEAR 毛玻璃模糊效果 + blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR) + + face_roi = frame[top:bottom, left:right] + + blurred_frame[top:bottom, left:right] = face_roi + black_bg[top:bottom, left:right] = face_roi + + return blurred_frame + + +frame_queue = queue.Queue(maxsize=2) + +video_queue = queue.Queue(maxsize=10) + +data_queue = queue.Queue(maxsize=10) + +show_queue = queue.Queue(maxsize=10) stop_event = threading.Event() + def capture_thread(): """ 采集线程：优化了分发逻辑，对视频流进行降频处理 """ cap = cv2.VideoCapture(0) - cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) - cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) - + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) + print("[Capture] 摄像头启动...") - + frame_count = 0 - + last_time = time.time() + while not stop_event.is_set(): ret, frame = cap.read() if not ret: break - + if not frame_queue.full(): frame_queue.put(frame) else: @@ -53,21 +115,27 @@ def capture_thread(): except queue.Empty: pass - - if frame_count % 2 == 0: - try: - if video_queue.full(): - video_queue.get_nowait() - video_queue.put(frame) - except: - pass - + # try: + # if video_queue.full(): + # video_queue.get_nowait() + # video_queue.put(frame) + # except: + # pass + frame_count += 1 - time.sleep(0.01) - + # time.sleep(1 / 30) + # current_time = time.time() + # if current_time - last_time >= 1.0: + # print(f"[Capture] FPS: {frame_count}") + # frame_count = 0 + # last_time = current_time + # print(current_time - last_time) + # last_time = current_time + cap.release() print("[Capture] 线程结束") + def analysis_thread(): """ 核心分析线程： @@ -76,15 +144,27 @@ def analysis_thread(): """ monitor = MonitorSystem(BASIC_FACE_DB) print("[Analysis] 分析系统启动...") - + freq = 0 + gap = 60 + status = 0 # 0:open 1:close + last_time = time.time() + last_freq = 0 + # heart_monitor = HeartRateMonitor() while not stop_event.is_set(): try: frame = frame_queue.get(timeout=1) except queue.Empty: continue - + # 核心分析 result = monitor.process_frame(frame) + result["eye_close_freq"] = 0 + result["heart_rate_bpm"] = 0 + + if video_queue.full(): + video_queue.get_nowait() + video_queue.put(result["frame"]) + payload = { "id": CAMERA_ID, @@ -92,91 +172,159 @@ def analysis_thread(): "name": "", "ear": "", "mar": "", + "iris_ratio": "", + "eye_close_freq": "", "pose": "", "emo_label": "", - "emo_va": "" + "emo_va": "", + "heart_rate_bpm": "", } if result["has_face"] and result["identity"]: - payload.update({ - "name": result["identity"]["name"], - "ear": result["ear"], - "mar": result["mar"], - "pose": result["pose"], - "emo_label": result["emotion_label"], - "emo_va": result["emotion_va"] - }) + payload.update( + { + "name": result["identity"]["name"], + "ear": result["ear"], + "mar": result["mar"], + "iris_ratio": result["iris_ratio"], + "pose": result["pose"], + "emo_label": result["emotion_label"], + "emo_va": result["emotion_va"], + } + ) elif result["has_face"]: - payload.update({ - "name": "Unknown", - "ear": result["ear"], - "mar": result["mar"], - "pose": result["pose"], - "emo_label": result["emotion_label"], - "emo_va": result["emotion_va"] - }) + payload.update( + { + "name": "Unknown", + "ear": result["ear"], + "mar": result["mar"], + "iris_ratio": result["iris_ratio"], + "pose": result["pose"], + "emo_label": result["emotion_label"], + "emo_va": result["emotion_va"], + } + ) + if result["has_face"] and result["ear"] < 0.2: + if status == 0: + freq += 1 + status = 1 + elif result["has_face"] and result["ear"] >= 0.2: + if status == 1: + freq += 1 + status = 0 + + if time.time() - last_time >= gap: + last_freq = freq / 2 + freq = 0 + last_time = time.time() + result["eye_close_freq"] = last_freq + payload["eye_close_freq"] = last_freq + # bpm = heart_monitor.process_frame(frame, result["landmark"]) + # if bpm != None: + # result["heart_rate_bpm"] = bpm + # payload["heart_rate_bpm"] = bpm if data_queue.full(): try: - _ = data_queue.get_nowait() + _ = data_queue.get_nowait() except queue.Empty: pass - + data_queue.put(payload) - draw_debug_info(frame, result) - cv2.imshow("Monitor Client", frame) - if cv2.waitKey(1) & 0xFF == ord('q'): - stop_event.set() + show_queue.put((result["frame"], result)) + # draw_debug_info(frame, result) + # cv2.imshow("Monitor Client", frame) - cv2.destroyAllWindows() print("[Analysis] 分析线程结束") + def video_stream_thread(): """ 发送线程：优化了 Socket 设置和压缩参数 """ print(f"[Video] 准备连接服务器 {SERVER_HOST}:{SERVER_PORT} ...") - + server = WebRTCServer(fps=60) + server.start() + fourcc = cv2.VideoWriter_fourcc(*'avc1') + # jetson-nvenc 编码器 + # bitrate = 1000000 # 1 Mbps + # gst_pipeline = ( + # f"appsrc ! " + # f"video/x-raw, format=BGR ! " + # f"queue ! " + # f"videoconvert ! " + # f"video/x-raw, format=RGBA ! " + # f"nvvidconv ! " + # f"nvv4l2h264enc bitrate={bitrate} control-rate=1 profile=High ! " + # f"h264parse ! " + # f"qtmux ! " + # f"filesink location={filename} " + # ) + # out = cv2.VideoWriter(gst_pipeline, cv2.CAP_GSTREAMER, 0, fps, (width, height)) + out1 = cv2.VideoWriter('output1.mp4', fourcc, 30.0, (1280, 720)) + out2 = cv2.VideoWriter('output2.mp4', fourcc, 30.0, (1280, 720)) while not stop_event.is_set(): try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - - s.connect((SERVER_HOST, SERVER_PORT)) - print(f"[Video] 已连接") - - camera_id_bytes = CAMERA_ID.encode('utf-8') - - while not stop_event.is_set(): - try: - frame = video_queue.get(timeout=1) - - small_frame = cv2.resize(frame, (320, 240)) - - ret, buffer = cv2.imencode('.jpg', small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50]) - - if not ret: continue - - frame_bytes = buffer.tobytes() - header_id_len = len(camera_id_bytes).to_bytes(4, 'big') - header_frame_len = len(frame_bytes).to_bytes(4, 'big') - - packet = header_id_len + camera_id_bytes + header_frame_len + frame_bytes - s.sendall(packet) - - except queue.Empty: - continue - except Exception as e: - print(f"[Video] 发送断开: {e}") - break - + frame = video_queue.get(timeout=1) + # small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720)) + server.provide_frame(frame) + out1.write(frame) + out2.write(frame) + except queue.Empty: + continue except Exception as e: - print(f"[Video] 重连中... {e}") - time.sleep(3) + print(f"[Video] 发送错误: {e}") + continue + # while not stop_event.is_set(): + # try: + # with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + # s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + # s.connect((SERVER_HOST, SERVER_PORT)) + # print(f"[Video] 已连接") + + # camera_id_bytes = CAMERA_ID.encode("utf-8") + + # while not stop_event.is_set(): + # try: + # frame = video_queue.get(timeout=1) + + # small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720)) + + # ret, buffer = cv2.imencode( + # ".jpg", small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50] + # ) + + # if not ret: + # continue + + # frame_bytes = buffer.tobytes() + # header_id_len = len(camera_id_bytes).to_bytes(4, "big") + # header_frame_len = len(frame_bytes).to_bytes(4, "big") + + # packet = ( + # header_id_len + # + camera_id_bytes + # + header_frame_len + # + frame_bytes + # ) + # s.sendall(packet) + + # except queue.Empty: + # continue + # except Exception as e: + # print(f"[Video] 发送断开: {e}") + # break + + # except Exception as e: + # print(f"[Video] 重连中... {e}") + # time.sleep(3) + out1.release() + out2.release() print("[Video] 线程结束") + def data_upload_thread(): """ 周期性爆发模式 @@ -185,9 +333,9 @@ def data_upload_thread(): """ print("[Data] 数据上报线程启动 (周期模式: 休眠30s -> 连发5次)") - LONG_SLEEP = 30 - BURST_COUNT = 5 - BURST_GAP = 1 + LONG_SLEEP = 30 + BURST_COUNT = 5 + BURST_GAP = 1 while not stop_event.is_set(): # --- 阶段 1: 长休眠 (30秒) --- @@ -196,7 +344,7 @@ def data_upload_thread(): # --- 阶段 2: 爆发发送 (5次) --- print(f"[Data] 开始上报周期 (连发 {BURST_COUNT} 次)...") - + try: while not data_queue.empty(): data_queue.get_nowait() @@ -210,63 +358,126 @@ def data_upload_thread(): break try: - data = data_queue.get(timeout=1.5) + data = data_queue.get(timeout=1.5) try: req = urllib.request.Request( url=API_URL, - data=json.dumps(data).encode('utf-8'), - headers={'Content-Type': 'application/json'}, - method='POST' + data=json.dumps(data).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", ) with urllib.request.urlopen(req, timeout=2) as resp: pass - + # 打印日志 - name_info = data['name'] if data['name'] else "NO-FACE" - print(f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}") - + name_info = data["name"] if data["name"] else "NO-FACE" + print( + f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}" + ) + except Exception as e: print(f"[Data] Upload Error: {e}") except queue.Empty: print(f"[Data] 队列为空，跳过第 {i+1} 次发送") - + if i < BURST_COUNT - 1: stop_event.wait(BURST_GAP) print("[Data] 数据上报线程结束") + def draw_debug_info(frame, result): """在画面上画出即时数据""" if not result["has_face"]: - cv2.putText(frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) + cv2.putText( + frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2 + ) return # 显示身份 id_text = result["identity"]["name"] if result["identity"] else "Unknown" color = (0, 255, 0) if result["identity"] else (0, 255, 255) - cv2.putText(frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2) - + cv2.putText( + frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2 + ) + # 显示数据 - cv2.putText(frame, f"EAR: {result['ear']}", (20, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1) - cv2.putText(frame, f"MAR: {result['mar']}", (20, 95), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1) - if result['ear'] < 0.15: - cv2.putText(frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) - + cv2.putText( + frame, + f"EAR: {result['ear']}", + (20, 70), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (255, 255, 0), + 1, + ) + cv2.putText( + frame, + f"MAR: {result['mar']}", + (20, 95), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (255, 255, 0), + 1, + ) + cv2.putText( + frame, + f"Iris Ratio: {result['iris_ratio']}", + (20, 190), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (255, 255, 0), + 1, + ) + cv2.putText( + frame, + f"Eye Close Freq: {result['eye_close_freq']}", + (20, 170), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (255, 0, 255), + 1, + ) + cv2.putText( + frame, + f"Heart Rate BPM: {result['heart_rate_bpm']}", + (20, 210), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (0, 165, 255), + 1, + ) + if result["ear"] < 0.15: + cv2.putText( + frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2 + ) + p, y, r = result["pose"] - cv2.putText(frame, f"Pose: P{p} Y{y} R{r}", (20, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 1) + cv2.putText( + frame, + f"Pose: P{p} Y{y} R{r}", + (20, 120), + cv2.FONT_HERSHEY_SIMPLEX, + 0.6, + (0, 255, 255), + 1, + ) emo = result.get("emotion_label", "N/A") - va = result.get("emotion_va", (0,0)) + va = result.get("emotion_va", (0, 0)) # 显示格式: Emo: happy (-0.5, 0.2) emo_text = f"Emo: {emo} ({va[0]:.2f}, {va[1]:.2f})" - cv2.putText(frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1) + cv2.putText( + frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1 + ) + if __name__ == "__main__": - t1 = threading.Thread(target=capture_thread, daemon=True) - t2 = threading.Thread(target=analysis_thread, daemon=True) + t1 = threading.Thread(target=capture_thread, daemon=True) + t2 = threading.Thread(target=analysis_thread, daemon=True) t3 = threading.Thread(target=video_stream_thread, daemon=True) - t4 = threading.Thread(target=data_upload_thread, daemon=True) + t4 = threading.Thread(target=data_upload_thread, daemon=True) t1.start() t2.start() @@ -275,7 +486,19 @@ if __name__ == "__main__": try: while not stop_event.is_set(): - time.sleep(1) + try: + frame, result = show_queue.get(timeout=1) + except queue.Empty: + continue + # frame = apply_soft_roi(frame) + display_frame = frame.copy() + draw_debug_info(display_frame, result) + cv2.imshow("Monitor Client", display_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + stop_event.set() + # time.sleep(1) + + cv2.destroyAllWindows() except KeyboardInterrupt: print("停止程序...") stop_event.set() @@ -283,4 +506,4 @@ if __name__ == "__main__": t1.join() t2.join() t3.join() - t4.join() \ No newline at end of file + t4.join() diff --git a/reproject/server_demo.py b/reproject/server_demo.py new file mode 100644 index 0000000..5962969 --- /dev/null +++ b/reproject/server_demo.py @@ -0,0 +1,49 @@ +import time +import numpy as np +import cv2 + +from webrtc_server import WebRTCServer + +FPS = 60 + +server = WebRTCServer(fps=FPS) +server.start() + + +def run_ani(): + width, height = 1920, 1080 + frame_count = 60 + while True: + frame = np.zeros((height, width, 3), dtype=np.uint8) + center_x = (frame_count * 10) % width + cv2.circle(frame, (center_x, height // 2), 50, (0, 255, 0), -1) + cv2.putText( + frame, + f"AniCam Frame: {frame_count}", + (10, 30), + cv2.FONT_HERSHEY_SIMPLEX, + 1, + (255, 255, 255), + 2, + ) + color = (frame_count * 5 % 256, 100, 200) + cv2.rectangle(frame, (50, 50), (150, 150), color, -1) + server.provide_frame(frame) + frame_count += 1 + time.sleep(1 / FPS) + + +def run_cam(device_id): + cap = cv2.VideoCapture(device_id) + cap.set(cv2.CAP_PROP_FRAME_WIDTH, 2560) + cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1440) + while True: + ret, frame = cap.read() + if ret: + server.provide_frame(frame) + cv2.imshow("Camera", frame) + time.sleep(1 / FPS) + + +if __name__ == "__main__": + run_cam(0) \ No newline at end of file diff --git a/reproject/test.py b/reproject/test.py new file mode 100644 index 0000000..e312c90 --- /dev/null +++ b/reproject/test.py @@ -0,0 +1,97 @@ +import cv2 +import threading +import time +import queue +import socket +import json +import urllib.request +import struct +import mediapipe as mp +import numpy as np +from analyzer import MonitorSystem + +mp_face_mesh = mp.solutions.face_mesh +face_mesh = mp_face_mesh.FaceMesh( + max_num_faces=1, + refine_landmarks=True, + min_detection_confidence=0.5, + min_tracking_confidence=0.5 +) + +def apply_soft_roi(frame): + h, w = frame.shape[:2] + rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + black_bg = np.zeros((h, w, 3), dtype=np.uint8) + results = face_mesh.process(rgb_frame) + if not results.multi_face_landmarks: + return frame + landmarks = results.multi_face_landmarks[0].landmark + xs = [l.x for l in landmarks] + ys = [l.y for l in landmarks] + # 计算人脸框 + face_loc = ( + int(min(ys) * h - 0.1 * h), int(max(xs) * w + 0.1 * w), + int(max(ys) * h + 0.1 * h), int(min(xs) * w - 0.1 * w) + ) + pad = 30 + face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad), + min(h, face_loc[2]+pad), max(0, face_loc[3]-pad)) + top = face_loc[0] + right = face_loc[1] + bottom = face_loc[2] + left = face_loc[3] + scale_factor = 10 + small_bg = cv2.resize(frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR) + # 使用 INTER_NEAREST 马赛克效果 + # 使用 INTER_LINEAR 毛玻璃模糊效果 + blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR) + + face_roi = frame[top:bottom, left:right] + + blurred_frame[top:bottom, left:right] = face_roi + black_bg[top:bottom, left:right] = face_roi + + return blurred_frame + +def test_compression_efficiency(): + # 1. 读取一张测试图 (或者用摄像头抓一帧) + cap = cv2.VideoCapture(0) + time.sleep(5) # 等待摄像头稳定 + while True: + ret, frame = cap.read() + cap.release() + + if not ret: + print("无法读取摄像头") + return + + # 2. 生成模糊处理后的图 + processed_frame = apply_soft_roi(frame) # 使用你的函数 + + # 3. 【关键】模拟网络传输/存储：进行 JPG 编码 + # 这里的 80 代表 JPEG 质量，模拟视频编码过程 + encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 80] + + # 编码原图 + _, encoded_original = cv2.imencode('.jpg', frame, encode_param) + original_size = len(encoded_original) + + # 编码处理后的图 + _, encoded_processed = cv2.imencode('.jpg', processed_frame, encode_param) + processed_size = len(encoded_processed) + + # 4. 对比结果 + print(f"原始画面编码后大小: {original_size / 1024:.2f} KB") + print(f"ROI处理编码后大小: {processed_size / 1024:.2f} KB") + + savings = (original_size - processed_size) / original_size * 100 + print(f"📉 带宽/存储节省了: {savings:.2f}%") + + # 可视化对比 + cv2.imshow("Original", frame) + cv2.imshow("Processed (Bandwidth Saver)", processed_frame) + + +# 运行测试 +# 注意：你需要先定义 face_mesh 和 apply_soft_roi 才能运行 +test_compression_efficiency() \ No newline at end of file diff --git a/reproject/webrtc_server.py b/reproject/webrtc_server.py new file mode 100644 index 0000000..f26a361 --- /dev/null +++ b/reproject/webrtc_server.py @@ -0,0 +1,157 @@ +import threading +import time +import asyncio +import socketio +import numpy as np +import cv2 +import av + +from aiortc import ( + MediaStreamError, + RTCConfiguration, + RTCPeerConnection, + RTCSessionDescription, + VideoStreamTrack, +) +from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE + + +class WebRTCServer: + def __init__(self, fps): + self.pcs = set() + self.fps = fps + self.frameContainer = [None] + + self.background_loop = asyncio.new_event_loop() + + self._rtc_thread = threading.Thread( + target=self._start_background_loop, + args=(self.background_loop,), + daemon=True, + ) + self._rtc_thread.start() + + self.server = "ws://10.128.50.6:5000" + + def _start_background_loop(self, loop): + asyncio.set_event_loop(loop) + loop.run_forever() + + async def _websocket_start(self): + sio = socketio.AsyncClient() + + @sio.event + async def connect(): + print("已连接到中心信令服务器") + # TODO 注册自己为设备 + await sio.emit("checkin", {"device_id": "cam_001"}) + + @sio.event + async def offer(data): + # data 包含: { sdp: "...", type: "offer", "sid": "..." } + print("收到 WebRTC Offer") + + localDescription = await self._handle_offer(data) + + await sio.emit( + "answer", + { + "sdp": localDescription.sdp, + "type": localDescription.type, + "sid": data["sid"], + }, + ) + + await sio.connect(self.server) + print(self.server, "connected") + await sio.wait() + + async def _handle_offer(self, offer): + pc = RTCPeerConnection(RTCConfiguration(iceServers=[])) + self.pcs.add(pc) + start = time.time() + + @pc.on("connectionstatechange") + async def on_state_change(): + if pc.connectionState == "failed" or pc.connectionState == "closed": + await pc.close() + print("Connection closed") + self.pcs.discard(pc) + + @pc.on("datachannel") + def on_data_channel(channel): + route_channel(channel) + + await pc.setRemoteDescription( + RTCSessionDescription(offer["sdp"], offer.get("type", "offer")) + ) + pc.addTrack(VideoFrameTrack(self.fps, self.frameContainer)) + + answer = await pc.createAnswer() + await pc.setLocalDescription(answer) + print(f"Handle offer in {(time.time() - start)*1000:.2f}ms") + return pc.localDescription + + def start(self): + asyncio.run_coroutine_threadsafe(self._websocket_start(), self.background_loop) + + def provide_frame(self, frame): + self.frameContainer[0] = frame + + def stop(self): + if self.background_loop.is_running(): + self.background_loop.call_soon_threadsafe(self.background_loop.stop) + if self._rtc_thread.is_alive(): + self._rtc_thread.join(timeout=2) + + +class VideoFrameTrack(VideoStreamTrack): + def __init__(self, fps, fc): + super().__init__() + self.fps = fps + self.frameContainer = fc + + async def next_timestamp(self): + """ + 重写父类方法，去除帧率限制 + """ + if self.readyState != "live": + raise MediaStreamError + + if hasattr(self, "_timestamp"): + self._timestamp += int(1 / self.fps * VIDEO_CLOCK_RATE) + wait = self._start + (self._timestamp / VIDEO_CLOCK_RATE) - time.time() + await asyncio.sleep(wait) + else: + self._start = time.time() + self._timestamp = 0 + return self._timestamp, VIDEO_TIME_BASE + + async def recv(self): + pts, time_base = await self.next_timestamp() + frame = self.frameContainer[0] + if frame is None: + frame = np.zeros((480, 640, 3), dtype=np.uint8) + else: + frame = self.frameContainer[0] + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + video_frame = av.VideoFrame.from_ndarray(frame, format="rgb24") + video_frame.pts = pts + video_frame.time_base = time_base + return video_frame + + +def route_channel(channel): + match channel.label: + case "latency": + + @channel.on("message") + def on_message(message): + now = int(time.time() * 1000 + 0.5) + channel.send(str(now)) + + pre = int(message) + print(f"Latency: {now - pre}ms") + + case _: + print(f"Unknown Channel {channel.label}")