眨眼频率、眼动分析、心率、视频录制

reproject/HeartRateMonitor.py

@@ -0,0 +1,139 @@
+import numpy as np
+import collections
+from scipy import signal
+
+class HeartRateMonitor:
+    def __init__(self, fps=30, window_size=300):
+        self.fps = fps
+        self.buffer_size = window_size
+        
+        # 存储 RGB 三个通道的均值
+        self.times = np.zeros(window_size)
+        self.r_buffer = collections.deque(maxlen=window_size)
+        self.g_buffer = collections.deque(maxlen=window_size)
+        self.b_buffer = collections.deque(maxlen=window_size)
+        
+        # 滤波器状态
+        self.bp_b, self.bp_a = self._create_bandpass_filter(0.75, 2.5, fps) # 45-150 BPM
+        
+        # 平滑结果用的
+        self.bpm_history = collections.deque(maxlen=10)
+
+    def _create_bandpass_filter(self, lowcut, highcut, fs, order=5):
+        """创建巴特沃斯带通滤波器"""
+        nyq = 0.5 * fs
+        low = lowcut / nyq
+        high = highcut / nyq
+        b, a = signal.butter(order, [low, high], btype='band')
+        return b, a
+
+    def _pos_algorithm(self, r, g, b):
+        """
+        POS (Plane-Orthogonal-to-Skin) 算法
+        比单纯的绿色通道法强在一个地方：抗运动干扰
+        """
+        # 1. 归一化 (除以均值)
+        # 加上 1e-6 防止除零
+        r_n = r / (np.mean(r) + 1e-6)
+        g_n = g / (np.mean(g) + 1e-6)
+        b_n = b / (np.mean(b) + 1e-6)
+        
+        # 2. 投影到色度平面 (Matplotlib 里的经典公式)
+        # S1 = G - B
+        # S2 = G + B - 2R
+        s1 = g_n - b_n
+        s2 = g_n + b_n - 2 * r_n
+        
+        # 3. Alpha 微调 (Alpha Tuning)
+        # 这一步是为了消除镜面反射带来的运动噪声
+        alpha = np.std(s1) / (np.std(s2) + 1e-6)
+        
+        # 4. 融合信号
+        h = s1 + alpha * s2
+        return h
+
+    def process_frame(self, frame, face_loc):
+        """
+        输入: 原始无损 frame (BGR), 人脸框 (top, right, bottom, left)
+        输出: BPM 数值 或 None (数据不够时)
+        """
+        top, right, bottom, left = face_loc
+        
+        # --- 1. ROI 提取与保护 ---
+        h_img, w_img = frame.shape[:2]
+        
+        # 缩小 ROI 范围：只取脸中心 50% 区域 (避开背景和边缘)
+        h_box = bottom - top
+        w_box = right - left
+        
+        # 修正 ROI 坐标
+        roi_top = int(max(0, top + h_box * 0.3))
+        roi_bottom = int(min(h_img, bottom - h_box * 0.3))
+        roi_left = int(max(0, left + w_box * 0.3))
+        roi_right = int(min(w_img, right - w_box * 0.3))
+        
+        roi = frame[roi_top:roi_bottom, roi_left:roi_right]
+        
+        if roi.size == 0:
+            return None
+
+        # --- 2. 提取 RGB 均值 ---
+        # OpenCV 是 BGR
+        b_mean = np.mean(roi[:, :, 0])
+        g_mean = np.mean(roi[:, :, 1])
+        r_mean = np.mean(roi[:, :, 2])
+        
+        self.r_buffer.append(r_mean)
+        self.g_buffer.append(g_mean)
+        self.b_buffer.append(b_mean)
+
+        # 数据不够，返回 None
+        if len(self.r_buffer) < self.buffer_size:
+            progress = int(len(self.r_buffer) / self.buffer_size * 100)
+            return None # 或者返回 progress 表示进度
+
+        # --- 3. 信号处理 (核心升级部分) ---
+        r = np.array(self.r_buffer)
+        g = np.array(self.g_buffer)
+        b = np.array(self.b_buffer)
+
+        # A. 使用 POS 算法融合三通道 (抗干扰)
+        pulse_signal = self._pos_algorithm(r, g, b)
+        
+        # B. 消除直流分量 (Detrending)
+        # 这一步去掉了光线缓慢变化的干扰
+        pulse_signal = signal.detrend(pulse_signal)
+        
+        # C. 带通滤波 (Bandpass Filter)
+        # 只保留 0.75Hz - 2.5Hz 的信号
+        pulse_signal = signal.filtfilt(self.bp_b, self.bp_a, pulse_signal)
+        
+        # --- 4. 频域分析 (FFT) ---
+        # 加汉宁窗 (减少频谱泄露)
+        window = np.hanning(len(pulse_signal))
+        pulse_signal_windowed = pulse_signal * window
+        
+        # FFT
+        fft_res = np.fft.rfft(pulse_signal_windowed)
+        freqs = np.fft.rfftfreq(len(pulse_signal), 1.0/self.fps)
+        mag = np.abs(fft_res)
+        
+        # D. 寻找峰值
+        # 限制频率范围 (45 BPM - 180 BPM)
+        interest_idx = np.where((freqs >= 0.75) & (freqs <= 3.0))
+        valid_freqs = freqs[interest_idx]
+        valid_mags = mag[interest_idx]
+        
+        if len(valid_mags) == 0:
+            return None
+
+        max_idx = np.argmax(valid_mags)
+        peak_freq = valid_freqs[max_idx]
+        bpm = peak_freq * 60.0
+        
+        # --- 5. 结果平滑 ---
+        # 防止数字乱跳
+        self.bpm_history.append(bpm)
+        avg_bpm = np.mean(self.bpm_history)
+        
+        return int(avg_bpm)

reproject/analyzer.py

@@ -3,17 +3,28 @@ import mediapipe as mp
 import time
 import numpy as np
 from collections import deque
-from geometry_utils import calculate_ear, calculate_mar_simple, estimate_head_pose, LEFT_EYE, RIGHT_EYE
+from geometry_utils import (
+    calculate_ear,
+    calculate_mar_simple,
+    calculate_iris_pos,
+    estimate_head_pose,
+    LEFT_EYE,
+    RIGHT_EYE,
+    LEFT_EYE_GAZE_IDXS,
+    RIGHT_EYE_GAZE_IDXS,
+)
 from face_library import FaceLibrary
 
 
 try:
     from new_emotion_test import analyze_emotion_with_hsemotion
+
     HAS_EMOTION_MODULE = True
 except ImportError:
     print("⚠️ 未找到 new_emotion_test.py，情绪功能将不可用")
     HAS_EMOTION_MODULE = False
 
+
 class MonitorSystem:
     def __init__(self, face_db):
         # 初始化 MediaPipe
@@ -22,7 +33,7 @@ class MonitorSystem:
             max_num_faces=1,
             refine_landmarks=True,
             min_detection_confidence=0.5,
-            min_tracking_confidence=0.5
+            min_tracking_confidence=0.5,
         )
 
         # 初始化人脸底库
@@ -42,12 +53,13 @@ class MonitorSystem:
         self.HISTORY_LEN = 5
         self.ear_history = deque(maxlen=self.HISTORY_LEN)
         self.mar_history = deque(maxlen=self.HISTORY_LEN)
+        self.iris_ratio_history = [
+            deque(maxlen=self.HISTORY_LEN),
+            deque(maxlen=self.HISTORY_LEN),
+        ]
 
         # 缓存上一次的检测结果
-        self.cached_emotion = {
+        self.cached_emotion = {"label": "detecting...", "va": (0.0, 0.0)}
-            "label": "detecting...", 
-            "va": (0.0, 0.0)
-        }
 
     def _get_smoothed_value(self, history, current_val):
         """内部函数：计算滑动平均值"""
@@ -69,15 +81,20 @@ class MonitorSystem:
             "has_face": False,
             "ear": 0.0,
             "mar": 0.0,
+            "iris_ratio": (0.5, 0.5),  # 0最左/上，1最右/下
             "pose": (0, 0, 0),
             "identity": self.current_user,
             "emotion_label": self.cached_emotion["label"],
-            "emotion_va": self.cached_emotion["va"]
+            "emotion_va": self.cached_emotion["va"],
+            "landmark": (0, w, h, 0),
+            "frame": frame,
         }
 
         if not results.multi_face_landmarks:
             self.ear_history.clear()
             self.mar_history.clear()
+            self.iris_ratio_history[0].clear()
+            self.iris_ratio_history[1].clear()
             return analysis_data
 
         analysis_data["has_face"] = True
@@ -95,33 +112,86 @@ class MonitorSystem:
         right = np.array([landmarks[308].x * w, landmarks[308].y * h])
         raw_mar = calculate_mar_simple(top, bottom, left, right)
 
+        # 计算虹膜位置
+        left_iris_ratio = calculate_iris_pos(landmarks, LEFT_EYE_GAZE_IDXS, w, h)
+        right_iris_ratio = calculate_iris_pos(landmarks, RIGHT_EYE_GAZE_IDXS, w, h)
+        raw_iris_ratio = (
+            (left_iris_ratio[0] + right_iris_ratio[0]) / 2.0,
+            (left_iris_ratio[1] + right_iris_ratio[1]) / 2.0,
+        )
+
         # --- 使用 History 进行数据平滑 ---
         smoothed_ear = self._get_smoothed_value(self.ear_history, raw_ear)
         smoothed_mar = self._get_smoothed_value(self.mar_history, raw_mar)
+        smoothed_iris_ratio = (
+            (self._get_smoothed_value(self.iris_ratio_history[0], raw_iris_ratio[0])),
+            (self._get_smoothed_value(self.iris_ratio_history[1], raw_iris_ratio[1])),
+        )
 
         # 计算头部姿态
         pitch, yaw, roll = estimate_head_pose(landmarks, w, h)
 
-        analysis_data.update({
+        analysis_data.update(
+            {
                 "ear": round(smoothed_ear, 4),
                 "mar": round(smoothed_mar, 4),
-            "pose": (int(pitch), int(yaw), int(roll))
+                "iris_ratio": (
-        })
+                    round(smoothed_iris_ratio[0], 4),
+                    round(smoothed_iris_ratio[1], 4),
+                ),
+                "pose": (int(pitch), int(yaw), int(roll)),
+            }
+        )
 
-        now = time.time()
-
-        # --- 身份识别 ---
-        if now - self.last_identity_check_time > self.IDENTITY_CHECK_INTERVAL:
         xs = [l.x for l in landmarks]
         ys = [l.y for l in landmarks]
         # 计算人脸框
         face_loc = (
-                int(min(ys) * h), int(max(xs) * w),
+            int(min(ys) * h - 0.1 * h),
-                int(max(ys) * h), int(min(xs) * w)
+            int(max(xs) * w + 0.1 * w),
+            int(max(ys) * h + 0.1 * h),
+            int(min(xs) * w - 0.1 * w),
         )
-            pad = 20
+        pad = 30
-            face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad), 
+        face_loc = (
-                        min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
+            max(0, face_loc[0] - pad),
+            min(w, face_loc[1] + pad),
+            min(h, face_loc[2] + pad),
+            max(0, face_loc[3] - pad),
+        )
+        analysis_data["landmark"] = face_loc
+
+        # --- ROI处理(对比选择在哪里实现) ---
+        top = face_loc[0]
+        right = face_loc[1]
+        bottom = face_loc[2]
+        left = face_loc[3]
+        scale_factor = 10
+        small_bg = cv2.resize(
+            frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR
+        )
+        # 使用 INTER_NEAREST 马赛克效果
+        # 使用 INTER_LINEAR 毛玻璃模糊效果
+        blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
+
+        face_roi = frame[top:bottom, left:right]
+
+        blurred_frame[top:bottom, left:right] = face_roi
+        analysis_data["frame"] = blurred_frame
+
+        now = time.time()
+        # --- 身份识别 ---
+        if now - self.last_identity_check_time > self.IDENTITY_CHECK_INTERVAL:
+            # xs = [l.x for l in landmarks]
+            # ys = [l.y for l in landmarks]
+            # # 计算人脸框
+            # face_loc = (
+            #     int(min(ys) * h), int(max(xs) * w),
+            #     int(max(ys) * h), int(min(xs) * w)
+            # )
+            # pad = 20
+            # face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad),
+            #             min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
 
             match_result = self.face_lib.identify(rgb_frame, face_location=face_loc)
             if match_result:
@@ -131,7 +201,9 @@ class MonitorSystem:
         analysis_data["identity"] = self.current_user
 
         # --- 情绪识别 ---
-        if HAS_EMOTION_MODULE and (now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL):
+        if HAS_EMOTION_MODULE and (
+            now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL
+        ):
             if results.multi_face_landmarks:
                 landmarks = results.multi_face_landmarks[0].landmark
                 xs = [l.x for l in landmarks]
@@ -159,7 +231,9 @@ class MonitorSystem:
 
                         if emo_results:
                             top_res = emo_results[0]
-                            self.cached_emotion["label"] = top_res.get("emotion", "unknown")
+                            self.cached_emotion["label"] = top_res.get(
+                                "emotion", "unknown"
+                            )
                             self.cached_emotion["va"] = top_res.get("vaVal", (0.0, 0.0))
 
                     except Exception as e:

reproject/geometry_utils.py

@@ -5,12 +5,18 @@ import cv2
 LEFT_EYE = [33, 160, 158, 133, 153, 144]
 # 右眼
 RIGHT_EYE = [362, 385, 387, 263, 373, 380]
+# 左眼虹膜关键点索引
+LEFT_EYE_GAZE_IDXS = [33, 133, 159, 145, 468]
+# 右眼虹膜关键点索引
+RIGHT_EYE_GAZE_IDXS = [263, 362, 386, 374, 473] 
 # 嘴唇 (内圈)
 LIPS = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, 311, 312, 13]
 
+
 def _euclidean_distance(point1, point2):
     return np.linalg.norm(point1 - point2)
 
+
 def calculate_ear(landmarks, width, height):
     """计算眼睛纵横比 EAR"""
     # 坐标转换
@@ -25,32 +31,71 @@ def calculate_ear(landmarks, width, height):
     ear = (v1 + v2) / (2.0 * h)
     return ear
 
+
+def calculate_iris_pos(landmarks, indices, width, height):
+    p_left = np.array([landmarks[indices[0]].x * width, landmarks[indices[0]].y * height])
+    p_right = np.array([landmarks[indices[1]].x * width, landmarks[indices[1]].y * height])
+    p_top = np.array([landmarks[indices[2]].x * width, landmarks[indices[2]].y * height])
+    p_bottom = np.array([landmarks[indices[3]].x * width, landmarks[indices[3]].y * height])
+    p_iris = np.array([landmarks[indices[4]].x * width, landmarks[indices[4]].y * height])
+
+    # 修改为欧几里得距离计算
+    eye_width = _euclidean_distance(p_left, p_right)
+    iris_left = _euclidean_distance(p_iris, p_left)
+    if eye_width < 1e-3: 
+        raw_x = 0.5
+    else :
+        raw_x = iris_left / eye_width
+    eye_height = _euclidean_distance(p_bottom, p_top)
+    iris_top = _euclidean_distance(p_iris, p_top)
+    if eye_height < 1e-3:
+        raw_y = 0.5
+    else:
+        raw_y = iris_top / eye_height
+
+    # x_min = 0.4
+    # x_max = 0.6
+    # ratio_x = abs(raw_x - x_min) / (x_max - x_min)
+    # y_min = 0.2
+    # y_max = 0.8
+    # ratio_y = abs(raw_y - y_min) / (y_max - y_min)
+    ratio_x = raw_x
+    ratio_y = raw_y
+    
+    return max(0.0, min(1.0, ratio_x)), max(0.0, min(1.0, ratio_y))
+
+
 def calculate_mar(landmarks, width, height):
     """计算嘴巴纵横比 MAR"""
     points = np.array([(p.x * width, p.y * height) for p in landmarks])
     pass
 
+
 def calculate_mar_simple(top, bottom, left, right):
     h = _euclidean_distance(top, bottom)
     w = _euclidean_distance(left, right)
     return h / w
 
+
 # geometry_utils.py 中的 estimate_head_pose 函数替换为以下内容
 
+
 def estimate_head_pose(landmarks, width, height):
     """
     计算头部姿态 (Pitch, Yaw, Roll)
     返回单位：角度 (Degree)
     """
     # 3D 模型点 (标准人脸模型)
-    model_points = np.array([
+    model_points = np.array(
+        [
             (0.0, 0.0, 0.0),  # Nose tip
             (0.0, -330.0, -65.0),  # Chin
             (-225.0, 170.0, -135.0),  # Left eye left corner
             (225.0, 170.0, -135.0),  # Right eye right corner
             (-150.0, -150.0, -125.0),  # Left Mouth corner
-        (150.0, -150.0, -125.0)      # Right mouth corner
+            (150.0, -150.0, -125.0),  # Right mouth corner
-    ])
+        ]
+    )
 
     # MediaPipe 对应的关键点索引
     idx_list = [1, 152, 33, 263, 61, 291]
@@ -65,9 +110,8 @@ def estimate_head_pose(landmarks, width, height):
     focal_length = width
     center = (width / 2, height / 2)
     camera_matrix = np.array(
-        [[focal_length, 0, center[0]],
+        [[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]],
-         [0, focal_length, center[1]],
+        dtype="double",
-         [0, 0, 1]], dtype="double"
     )
     dist_coeffs = np.zeros((4, 1))
 
@@ -83,12 +127,18 @@ def estimate_head_pose(landmarks, width, height):
     yaw = angles[1]
     roll = angles[2]
 
-    if pitch < -180: pitch += 360
+    if pitch < -180:
-    if pitch > 180:  pitch -= 360
+        pitch += 360
+    if pitch > 180:
+        pitch -= 360
     pitch = 180 - pitch if pitch > 0 else -pitch - 180
-    if yaw < -180: yaw += 360
+    if yaw < -180:
-    if yaw > 180:  yaw -= 360
+        yaw += 360
-    if roll < -180: roll += 360
+    if yaw > 180:
-    if roll > 180:  roll -= 360
+        yaw -= 360
+    if roll < -180:
+        roll += 360
+    if roll > 180:
+        roll -= 360
 
     return pitch, yaw, roll

reproject/main.py

@@ -1,3 +1,4 @@
+from calendar import c
 import cv2
 import threading
 import time
@@ -6,9 +7,13 @@ import socket
 import json
 import urllib.request
 import struct
+import numpy as np
+import mediapipe as mp
 from analyzer import MonitorSystem
+from webrtc_server import WebRTCServer
+from HeartRateMonitor import HeartRateMonitor
 
-SERVER_HOST = '10.128.50.6'  
+SERVER_HOST = "10.128.50.6"
 SERVER_PORT = 65432
 API_URL = "http://10.128.50.6:5000/api/states"
 CAMERA_ID = "23373333"
@@ -18,26 +23,83 @@ BASIC_FACE_DB = {
     "Yaoyu": {"name": "Yaoyu Zhang", "age": 20, "image-path": "yaoyu.jpg"},
 }
 
+mp_face_mesh_main = mp.solutions.face_mesh
+face_mesh_main = mp_face_mesh_main.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=True,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5,
+)
+
+
+def apply_soft_roi(frame):
+    h, w = frame.shape[:2]
+    rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    black_bg = np.zeros((h, w, 3), dtype=np.uint8)
+
+    results = face_mesh_main.process(rgb_frame)
+    if not results.multi_face_landmarks:
+        return frame
+    landmarks = results.multi_face_landmarks[0].landmark
+    xs = [l.x for l in landmarks]
+    ys = [l.y for l in landmarks]
+    # 计算人脸框
+    face_loc = (
+        int(min(ys) * h - 0.1 * h),
+        int(max(xs) * w + 0.1 * w),
+        int(max(ys) * h + 0.1 * h),
+        int(min(xs) * w - 0.1 * w),
+    )
+    pad = 30
+    face_loc = (
+        max(0, face_loc[0] - pad),
+        min(w, face_loc[1] + pad),
+        min(h, face_loc[2] + pad),
+        max(0, face_loc[3] - pad),
+    )
+    top = face_loc[0]
+    right = face_loc[1]
+    bottom = face_loc[2]
+    left = face_loc[3]
+    scale_factor = 10
+    small_bg = cv2.resize(
+        frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR
+    )
+    # 使用 INTER_NEAREST 马赛克效果
+    # 使用 INTER_LINEAR 毛玻璃模糊效果
+    blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
+
+    face_roi = frame[top:bottom, left:right]
+
+    blurred_frame[top:bottom, left:right] = face_roi
+    black_bg[top:bottom, left:right] = face_roi
+
+    return blurred_frame
+
 
 frame_queue = queue.Queue(maxsize=2)
 
-video_queue = queue.Queue(maxsize=1)
+video_queue = queue.Queue(maxsize=10)
 
 data_queue = queue.Queue(maxsize=10)
 
+show_queue = queue.Queue(maxsize=10)
+
 stop_event = threading.Event()
 
+
 def capture_thread():
     """
     采集线程：优化了分发逻辑，对视频流进行降频处理
     """
     cap = cv2.VideoCapture(0)
-    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
+    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
-    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
+    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
 
     print("[Capture] 摄像头启动...")
 
     frame_count = 0
+    last_time = time.time()
 
     while not stop_event.is_set():
         ret, frame = cap.read()
@@ -53,21 +115,27 @@ def capture_thread():
             except queue.Empty:
                 pass
 
-       
+        # try:
-        if frame_count % 2 == 0:
+        #     if video_queue.full():
-            try:
+        #         video_queue.get_nowait()
-                if video_queue.full():
+        #     video_queue.put(frame)
-                    video_queue.get_nowait() 
+        # except:
-                video_queue.put(frame)
+        #     pass
-            except:
-                pass
 
         frame_count += 1
-        time.sleep(0.01)
+        # time.sleep(1 / 30)
+        # current_time = time.time()
+        # if current_time  - last_time >= 1.0:
+        #     print(f"[Capture] FPS: {frame_count}")
+        #     frame_count = 0
+        #     last_time = current_time
+        # print(current_time - last_time)
+        # last_time = current_time
 
     cap.release()
     print("[Capture] 线程结束")
 
+
 def analysis_thread():
     """
     核心分析线程：
@@ -76,7 +144,12 @@ def analysis_thread():
     """
     monitor = MonitorSystem(BASIC_FACE_DB)
     print("[Analysis] 分析系统启动...")
-    
+    freq = 0
+    gap = 60
+    status = 0  # 0:open 1:close
+    last_time = time.time()
+    last_freq = 0
+    # heart_monitor = HeartRateMonitor()
     while not stop_event.is_set():
         try:
             frame = frame_queue.get(timeout=1)
@@ -85,6 +158,13 @@ def analysis_thread():
 
         # 核心分析
         result = monitor.process_frame(frame)
+        result["eye_close_freq"] = 0
+        result["heart_rate_bpm"] = 0
+
+        if video_queue.full():
+            video_queue.get_nowait()
+        video_queue.put(result["frame"])
+            
 
         payload = {
             "id": CAMERA_ID,
@@ -92,30 +172,58 @@ def analysis_thread():
             "name": "",
             "ear": "",
             "mar": "",
+            "iris_ratio": "",
+            "eye_close_freq": "",
             "pose": "",
             "emo_label": "",
-            "emo_va": ""
+            "emo_va": "",
+            "heart_rate_bpm": "",
         }
 
         if result["has_face"] and result["identity"]:
-            payload.update({
+            payload.update(
+                {
                     "name": result["identity"]["name"],
                     "ear": result["ear"],
                     "mar": result["mar"],
+                    "iris_ratio": result["iris_ratio"],
                     "pose": result["pose"],
                     "emo_label": result["emotion_label"],
-                "emo_va": result["emotion_va"]
+                    "emo_va": result["emotion_va"],
-            })
+                }
+            )
         elif result["has_face"]:
-             payload.update({
+            payload.update(
+                {
                     "name": "Unknown",
                     "ear": result["ear"],
                     "mar": result["mar"],
+                    "iris_ratio": result["iris_ratio"],
                     "pose": result["pose"],
                     "emo_label": result["emotion_label"],
-                "emo_va": result["emotion_va"]
+                    "emo_va": result["emotion_va"],
-            })
+                }
+            )
 
+        if result["has_face"] and result["ear"] < 0.2:
+            if status == 0:
+                freq += 1
+            status = 1
+        elif result["has_face"] and result["ear"] >= 0.2:
+            if status == 1:
+                freq += 1
+            status = 0
+
+        if time.time() - last_time >= gap:
+            last_freq = freq / 2
+            freq = 0
+            last_time = time.time()
+        result["eye_close_freq"] = last_freq
+        payload["eye_close_freq"] = last_freq
+        # bpm = heart_monitor.process_frame(frame, result["landmark"])
+        # if bpm != None:
+        #     result["heart_rate_bpm"] = bpm
+        #     payload["heart_rate_bpm"] = bpm
         if data_queue.full():
             try:
                 _ = data_queue.get_nowait()
@@ -124,59 +232,99 @@ def analysis_thread():
 
         data_queue.put(payload)
 
-        draw_debug_info(frame, result)
+        show_queue.put((result["frame"], result))
-        cv2.imshow("Monitor Client", frame)
+        # draw_debug_info(frame, result)
-        if cv2.waitKey(1) & 0xFF == ord('q'):
+        # cv2.imshow("Monitor Client", frame)
-            stop_event.set()
 
-    cv2.destroyAllWindows()
     print("[Analysis] 分析线程结束")
 
+
 def video_stream_thread():
     """
     发送线程：优化了 Socket 设置和压缩参数
     """
     print(f"[Video] 准备连接服务器 {SERVER_HOST}:{SERVER_PORT} ...")
-    
+    server = WebRTCServer(fps=60)
-    while not stop_event.is_set():
+    server.start()
-        try:
+    fourcc = cv2.VideoWriter_fourcc(*'avc1')
-            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    # jetson-nvenc 编码器
-                s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+    # bitrate = 1000000  # 1 Mbps
-                
+    # gst_pipeline = (
-                s.connect((SERVER_HOST, SERVER_PORT))
+    #     f"appsrc ! "
-                print(f"[Video] 已连接")
+    #     f"video/x-raw, format=BGR ! "
-                
+    #     f"queue ! "
-                camera_id_bytes = CAMERA_ID.encode('utf-8')
+    #     f"videoconvert ! "
-                
+    #     f"video/x-raw, format=RGBA ! "
+    #     f"nvvidconv ! "
+    #     f"nvv4l2h264enc bitrate={bitrate} control-rate=1 profile=High ! "
+    #     f"h264parse ! "
+    #     f"qtmux ! "
+    #     f"filesink location={filename} "
+    # )
+    # out = cv2.VideoWriter(gst_pipeline, cv2.CAP_GSTREAMER, 0, fps, (width, height))
+    out1 = cv2.VideoWriter('output1.mp4', fourcc, 30.0, (1280, 720))
+    out2 = cv2.VideoWriter('output2.mp4', fourcc, 30.0, (1280, 720))
     while not stop_event.is_set():
         try:
             frame = video_queue.get(timeout=1)
-                        
+            # small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
-                        small_frame = cv2.resize(frame, (320, 240))
+            server.provide_frame(frame)
-                        
+            out1.write(frame)
-                        ret, buffer = cv2.imencode('.jpg', small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50])
+            out2.write(frame)
-                        
-                        if not ret: continue
-                        
-                        frame_bytes = buffer.tobytes()
-                        header_id_len = len(camera_id_bytes).to_bytes(4, 'big')
-                        header_frame_len = len(frame_bytes).to_bytes(4, 'big')
-                        
-                        packet = header_id_len + camera_id_bytes + header_frame_len + frame_bytes
-                        s.sendall(packet)
-                        
         except queue.Empty:
             continue
         except Exception as e:
-                        print(f"[Video] 发送断开: {e}")
+            print(f"[Video] 发送错误: {e}")
-                        break 
+            continue
+    # while not stop_event.is_set():
+    #     try:
+    #         with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+    #             s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
 
-        except Exception as e:
+    #             s.connect((SERVER_HOST, SERVER_PORT))
-            print(f"[Video] 重连中... {e}")
+    #             print(f"[Video] 已连接")
-            time.sleep(3)
 
+    #             camera_id_bytes = CAMERA_ID.encode("utf-8")
+
+    #             while not stop_event.is_set():
+    #                 try:
+    #                     frame = video_queue.get(timeout=1)
+
+    #                     small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
+
+    #                     ret, buffer = cv2.imencode(
+    #                         ".jpg", small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50]
+    #                     )
+
+    #                     if not ret:
+    #                         continue
+
+    #                     frame_bytes = buffer.tobytes()
+    #                     header_id_len = len(camera_id_bytes).to_bytes(4, "big")
+    #                     header_frame_len = len(frame_bytes).to_bytes(4, "big")
+
+    #                     packet = (
+    #                         header_id_len
+    #                         + camera_id_bytes
+    #                         + header_frame_len
+    #                         + frame_bytes
+    #                     )
+    #                     s.sendall(packet)
+
+    #                 except queue.Empty:
+    #                     continue
+    #                 except Exception as e:
+    #                     print(f"[Video] 发送断开: {e}")
+    #                     break
+
+    #     except Exception as e:
+    #         print(f"[Video] 重连中... {e}")
+    #         time.sleep(3)
+    out1.release()
+    out2.release()
     print("[Video] 线程结束")
 
+
 def data_upload_thread():
     """
     周期性爆发模式
@@ -214,16 +362,18 @@ def data_upload_thread():
                 try:
                     req = urllib.request.Request(
                         url=API_URL,
-                        data=json.dumps(data).encode('utf-8'),
+                        data=json.dumps(data).encode("utf-8"),
-                        headers={'Content-Type': 'application/json'},
+                        headers={"Content-Type": "application/json"},
-                        method='POST'
+                        method="POST",
                     )
                     with urllib.request.urlopen(req, timeout=2) as resp:
                         pass
 
                     # 打印日志
-                    name_info = data['name'] if data['name'] else "NO-FACE"
+                    name_info = data["name"] if data["name"] else "NO-FACE"
-                    print(f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}")
+                    print(
+                        f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}"
+                    )
 
                 except Exception as e:
                     print(f"[Data] Upload Error: {e}")
@@ -236,31 +386,92 @@ def data_upload_thread():
 
     print("[Data] 数据上报线程结束")
 
+
 def draw_debug_info(frame, result):
     """在画面上画出即时数据"""
     if not result["has_face"]:
-        cv2.putText(frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+        cv2.putText(
+            frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
+        )
         return
 
     # 显示身份
     id_text = result["identity"]["name"] if result["identity"] else "Unknown"
     color = (0, 255, 0) if result["identity"] else (0, 255, 255)
-    cv2.putText(frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2)
+    cv2.putText(
+        frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2
+    )
 
     # 显示数据
-    cv2.putText(frame, f"EAR: {result['ear']}", (20, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
+    cv2.putText(
-    cv2.putText(frame, f"MAR: {result['mar']}", (20, 95), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
+        frame,
-    if result['ear'] < 0.15:
+        f"EAR: {result['ear']}",
-        cv2.putText(frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+        (20, 70),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (255, 255, 0),
+        1,
+    )
+    cv2.putText(
+        frame,
+        f"MAR: {result['mar']}",
+        (20, 95),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (255, 255, 0),
+        1,
+    )
+    cv2.putText(
+        frame,
+        f"Iris Ratio: {result['iris_ratio']}",
+        (20, 190),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (255, 255, 0),
+        1,
+    )
+    cv2.putText(
+        frame,
+        f"Eye Close Freq: {result['eye_close_freq']}",
+        (20, 170),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (255, 0, 255),
+        1,
+    )
+    cv2.putText(
+        frame,
+        f"Heart Rate BPM: {result['heart_rate_bpm']}",
+        (20, 210),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (0, 165, 255),
+        1,
+    )
+    if result["ear"] < 0.15:
+        cv2.putText(
+            frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
+        )
 
     p, y, r = result["pose"]
-    cv2.putText(frame, f"Pose: P{p} Y{y} R{r}", (20, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 1)
+    cv2.putText(
+        frame,
+        f"Pose: P{p} Y{y} R{r}",
+        (20, 120),
+        cv2.FONT_HERSHEY_SIMPLEX,
+        0.6,
+        (0, 255, 255),
+        1,
+    )
 
     emo = result.get("emotion_label", "N/A")
-    va = result.get("emotion_va", (0,0))
+    va = result.get("emotion_va", (0, 0))
     # 显示格式: Emo: happy (-0.5, 0.2)
     emo_text = f"Emo: {emo} ({va[0]:.2f}, {va[1]:.2f})"
-    cv2.putText(frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1)
+    cv2.putText(
+        frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1
+    )
+
 
 if __name__ == "__main__":
     t1 = threading.Thread(target=capture_thread, daemon=True)
@@ -275,7 +486,19 @@ if __name__ == "__main__":
 
     try:
         while not stop_event.is_set():
-            time.sleep(1)
+            try:
+                frame, result = show_queue.get(timeout=1)
+            except queue.Empty:
+                continue
+            # frame = apply_soft_roi(frame)
+            display_frame = frame.copy()
+            draw_debug_info(display_frame, result)
+            cv2.imshow("Monitor Client", display_frame)
+            if cv2.waitKey(1) & 0xFF == ord("q"):
+                stop_event.set()
+            # time.sleep(1)
+        
+        cv2.destroyAllWindows()
     except KeyboardInterrupt:
         print("停止程序...")
         stop_event.set()

reproject/server_demo.py

@@ -0,0 +1,49 @@
+import time
+import numpy as np
+import cv2
+
+from webrtc_server import WebRTCServer
+
+FPS = 60
+
+server = WebRTCServer(fps=FPS)
+server.start()
+
+
+def run_ani():
+    width, height = 1920, 1080
+    frame_count = 60
+    while True:
+        frame = np.zeros((height, width, 3), dtype=np.uint8)
+        center_x = (frame_count * 10) % width
+        cv2.circle(frame, (center_x, height // 2), 50, (0, 255, 0), -1)
+        cv2.putText(
+            frame,
+            f"AniCam Frame: {frame_count}",
+            (10, 30),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            1,
+            (255, 255, 255),
+            2,
+        )
+        color = (frame_count * 5 % 256, 100, 200)
+        cv2.rectangle(frame, (50, 50), (150, 150), color, -1)
+        server.provide_frame(frame)
+        frame_count += 1
+        time.sleep(1 / FPS)
+
+
+def run_cam(device_id):
+    cap = cv2.VideoCapture(device_id)
+    cap.set(cv2.CAP_PROP_FRAME_WIDTH, 2560)
+    cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1440)
+    while True:
+        ret, frame = cap.read()
+        if ret:
+            server.provide_frame(frame)
+            cv2.imshow("Camera", frame)
+        time.sleep(1 / FPS)
+
+
+if __name__ == "__main__":
+    run_cam(0)

reproject/test.py

@@ -0,0 +1,97 @@
+import cv2
+import threading
+import time
+import queue
+import socket
+import json
+import urllib.request
+import struct 
+import mediapipe as mp
+import numpy as np
+from analyzer import MonitorSystem
+
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(
+    max_num_faces=1,
+    refine_landmarks=True,
+    min_detection_confidence=0.5,
+    min_tracking_confidence=0.5
+)
+
+def apply_soft_roi(frame):
+    h, w = frame.shape[:2]
+    rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    black_bg = np.zeros((h, w, 3), dtype=np.uint8)
+    results = face_mesh.process(rgb_frame)
+    if not results.multi_face_landmarks:
+        return frame
+    landmarks = results.multi_face_landmarks[0].landmark
+    xs = [l.x for l in landmarks]
+    ys = [l.y for l in landmarks]
+    # 计算人脸框
+    face_loc = (
+        int(min(ys) * h - 0.1 * h), int(max(xs) * w + 0.1 * w),
+        int(max(ys) * h + 0.1 * h), int(min(xs) * w - 0.1 * w)
+    )
+    pad = 30
+    face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad), 
+                min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
+    top = face_loc[0]
+    right = face_loc[1]
+    bottom = face_loc[2]
+    left = face_loc[3]
+    scale_factor = 10 
+    small_bg = cv2.resize(frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR)
+    # 使用 INTER_NEAREST 马赛克效果
+    # 使用 INTER_LINEAR 毛玻璃模糊效果
+    blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
+    
+    face_roi = frame[top:bottom, left:right]
+    
+    blurred_frame[top:bottom, left:right] = face_roi
+    black_bg[top:bottom, left:right] = face_roi
+
+    return blurred_frame
+
+def test_compression_efficiency():
+    # 1. 读取一张测试图 (或者用摄像头抓一帧)
+    cap = cv2.VideoCapture(0)
+    time.sleep(5)  # 等待摄像头稳定
+    while True:
+        ret, frame = cap.read()
+        cap.release()
+        
+        if not ret:
+            print("无法读取摄像头")
+            return
+
+        # 2. 生成模糊处理后的图
+        processed_frame = apply_soft_roi(frame) # 使用你的函数
+
+        # 3. 【关键】模拟网络传输/存储：进行 JPG 编码
+        # 这里的 80 代表 JPEG 质量，模拟视频编码过程
+        encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 80]
+        
+        # 编码原图
+        _, encoded_original = cv2.imencode('.jpg', frame, encode_param)
+        original_size = len(encoded_original)
+        
+        # 编码处理后的图
+        _, encoded_processed = cv2.imencode('.jpg', processed_frame, encode_param)
+        processed_size = len(encoded_processed)
+
+        # 4. 对比结果
+        print(f"原始画面编码后大小: {original_size / 1024:.2f} KB")
+        print(f"ROI处理编码后大小: {processed_size / 1024:.2f} KB")
+        
+        savings = (original_size - processed_size) / original_size * 100
+        print(f"📉 带宽/存储节省了: {savings:.2f}%")
+
+        # 可视化对比
+        cv2.imshow("Original", frame)
+        cv2.imshow("Processed (Bandwidth Saver)", processed_frame)
+        
+
+# 运行测试
+# 注意：你需要先定义 face_mesh 和 apply_soft_roi 才能运行
+test_compression_efficiency()

reproject/webrtc_server.py

@@ -0,0 +1,157 @@
+import threading
+import time
+import asyncio
+import socketio
+import numpy as np
+import cv2
+import av
+
+from aiortc import (
+    MediaStreamError,
+    RTCConfiguration,
+    RTCPeerConnection,
+    RTCSessionDescription,
+    VideoStreamTrack,
+)
+from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
+
+
+class WebRTCServer:
+    def __init__(self, fps):
+        self.pcs = set()
+        self.fps = fps
+        self.frameContainer = [None]
+
+        self.background_loop = asyncio.new_event_loop()
+
+        self._rtc_thread = threading.Thread(
+            target=self._start_background_loop,
+            args=(self.background_loop,),
+            daemon=True,
+        )
+        self._rtc_thread.start()
+
+        self.server = "ws://10.128.50.6:5000"
+
+    def _start_background_loop(self, loop):
+        asyncio.set_event_loop(loop)
+        loop.run_forever()
+
+    async def _websocket_start(self):
+        sio = socketio.AsyncClient()
+
+        @sio.event
+        async def connect():
+            print("已连接到中心信令服务器")
+            # TODO 注册自己为设备
+            await sio.emit("checkin", {"device_id": "cam_001"})
+
+        @sio.event
+        async def offer(data):
+            # data 包含: { sdp: "...", type: "offer", "sid": "..." }
+            print("收到 WebRTC Offer")
+
+            localDescription = await self._handle_offer(data)
+
+            await sio.emit(
+                "answer",
+                {
+                    "sdp": localDescription.sdp,
+                    "type": localDescription.type,
+                    "sid": data["sid"],
+                },
+            )
+
+        await sio.connect(self.server)
+        print(self.server, "connected")
+        await sio.wait()
+
+    async def _handle_offer(self, offer):
+        pc = RTCPeerConnection(RTCConfiguration(iceServers=[]))
+        self.pcs.add(pc)
+        start = time.time()
+
+        @pc.on("connectionstatechange")
+        async def on_state_change():
+            if pc.connectionState == "failed" or pc.connectionState == "closed":
+                await pc.close()
+                print("Connection closed")
+                self.pcs.discard(pc)
+
+        @pc.on("datachannel")
+        def on_data_channel(channel):
+            route_channel(channel)
+
+        await pc.setRemoteDescription(
+            RTCSessionDescription(offer["sdp"], offer.get("type", "offer"))
+        )
+        pc.addTrack(VideoFrameTrack(self.fps, self.frameContainer))
+
+        answer = await pc.createAnswer()
+        await pc.setLocalDescription(answer)
+        print(f"Handle offer in {(time.time() - start)*1000:.2f}ms")
+        return pc.localDescription
+
+    def start(self):
+        asyncio.run_coroutine_threadsafe(self._websocket_start(), self.background_loop)
+
+    def provide_frame(self, frame):
+        self.frameContainer[0] = frame
+
+    def stop(self):
+        if self.background_loop.is_running():
+            self.background_loop.call_soon_threadsafe(self.background_loop.stop)
+        if self._rtc_thread.is_alive():
+            self._rtc_thread.join(timeout=2)
+
+
+class VideoFrameTrack(VideoStreamTrack):
+    def __init__(self, fps, fc):
+        super().__init__()
+        self.fps = fps
+        self.frameContainer = fc
+
+    async def next_timestamp(self):
+        """
+        重写父类方法，去除帧率限制
+        """
+        if self.readyState != "live":
+            raise MediaStreamError
+
+        if hasattr(self, "_timestamp"):
+            self._timestamp += int(1 / self.fps * VIDEO_CLOCK_RATE)
+            wait = self._start + (self._timestamp / VIDEO_CLOCK_RATE) - time.time()
+            await asyncio.sleep(wait)
+        else:
+            self._start = time.time()
+            self._timestamp = 0
+        return self._timestamp, VIDEO_TIME_BASE
+
+    async def recv(self):
+        pts, time_base = await self.next_timestamp()
+        frame = self.frameContainer[0]
+        if frame is None:
+            frame = np.zeros((480, 640, 3), dtype=np.uint8)
+        else:
+            frame = self.frameContainer[0]
+            frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        video_frame = av.VideoFrame.from_ndarray(frame, format="rgb24")
+        video_frame.pts = pts
+        video_frame.time_base = time_base
+        return video_frame
+
+
+def route_channel(channel):
+    match channel.label:
+        case "latency":
+
+            @channel.on("message")
+            def on_message(message):
+                now = int(time.time() * 1000 + 0.5)
+                channel.send(str(now))
+
+                pre = int(message)
+                print(f"Latency: {now - pre}ms")
+
+        case _:
+            print(f"Unknown Channel {channel.label}")