colden/cv_state_ana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

眨眼频率、眼动分析、心率、视频录制

Browse Source
This commit is contained in:
邓智航
2026-01-25 15:19:01 +08:00
parent 6e882d2aa4
commit b3997c2646
7 changed files with 977 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
reproject/HeartRateMonitor.py Normal file
View File

@@ -0,0 +1,139 @@
import numpy as np
import collections
from scipy import signal
class HeartRateMonitor:
def __init__(self, fps=30, window_size=300):
self.fps = fps
self.buffer_size = window_size
# 存储 RGB 三个通道的均值
self.times = np.zeros(window_size)
self.r_buffer = collections.deque(maxlen=window_size)
self.g_buffer = collections.deque(maxlen=window_size)
self.b_buffer = collections.deque(maxlen=window_size)
# 滤波器状态
self.bp_b, self.bp_a = self._create_bandpass_filter(0.75, 2.5, fps) # 45-150 BPM
# 平滑结果用的
self.bpm_history = collections.deque(maxlen=10)
def _create_bandpass_filter(self, lowcut, highcut, fs, order=5):
"""创建巴特沃斯带通滤波器"""
nyq = 0.5 * fs
low = lowcut / nyq
high = highcut / nyq
b, a = signal.butter(order, [low, high], btype='band')
return b, a
def _pos_algorithm(self, r, g, b):
"""
POS (Plane-Orthogonal-to-Skin) 算法
比单纯的绿色通道法强在一个地方：抗运动干扰
"""
# 1. 归一化 (除以均值)
# 加上 1e-6 防止除零
r_n = r / (np.mean(r) + 1e-6)
g_n = g / (np.mean(g) + 1e-6)
b_n = b / (np.mean(b) + 1e-6)
# 2. 投影到色度平面 (Matplotlib 里的经典公式)
# S1 = G - B
# S2 = G + B - 2R
s1 = g_n - b_n
s2 = g_n + b_n - 2 * r_n
# 3. Alpha 微调 (Alpha Tuning)
# 这一步是为了消除镜面反射带来的运动噪声
alpha = np.std(s1) / (np.std(s2) + 1e-6)
# 4. 融合信号
h = s1 + alpha * s2
return h
def process_frame(self, frame, face_loc):
"""
输入: 原始无损 frame (BGR), 人脸框 (top, right, bottom, left)
输出: BPM 数值 或 None (数据不够时)
"""
top, right, bottom, left = face_loc
# --- 1. ROI 提取与保护 ---
h_img, w_img = frame.shape[:2]
# 缩小 ROI 范围：只取脸中心 50% 区域 (避开背景和边缘)
h_box = bottom - top
w_box = right - left
# 修正 ROI 坐标
roi_top = int(max(0, top + h_box * 0.3))
roi_bottom = int(min(h_img, bottom - h_box * 0.3))
roi_left = int(max(0, left + w_box * 0.3))
roi_right = int(min(w_img, right - w_box * 0.3))
roi = frame[roi_top:roi_bottom, roi_left:roi_right]
if roi.size == 0:
return None
# --- 2. 提取 RGB 均值 ---
# OpenCV 是 BGR
b_mean = np.mean(roi[:, :, 0])
g_mean = np.mean(roi[:, :, 1])
r_mean = np.mean(roi[:, :, 2])
self.r_buffer.append(r_mean)
self.g_buffer.append(g_mean)
self.b_buffer.append(b_mean)
# 数据不够，返回 None
if len(self.r_buffer) < self.buffer_size:
progress = int(len(self.r_buffer) / self.buffer_size * 100)
return None # 或者返回 progress 表示进度
# --- 3. 信号处理 (核心升级部分) ---
r = np.array(self.r_buffer)
g = np.array(self.g_buffer)
b = np.array(self.b_buffer)
# A. 使用 POS 算法融合三通道 (抗干扰)
pulse_signal = self._pos_algorithm(r, g, b)
# B. 消除直流分量 (Detrending)
# 这一步去掉了光线缓慢变化的干扰
pulse_signal = signal.detrend(pulse_signal)
# C. 带通滤波 (Bandpass Filter)
# 只保留 0.75Hz - 2.5Hz 的信号
pulse_signal = signal.filtfilt(self.bp_b, self.bp_a, pulse_signal)
# --- 4. 频域分析 (FFT) ---
# 加汉宁窗 (减少频谱泄露)
window = np.hanning(len(pulse_signal))
pulse_signal_windowed = pulse_signal * window
# FFT
fft_res = np.fft.rfft(pulse_signal_windowed)
freqs = np.fft.rfftfreq(len(pulse_signal), 1.0/self.fps)
mag = np.abs(fft_res)
# D. 寻找峰值
# 限制频率范围 (45 BPM - 180 BPM)
interest_idx = np.where((freqs >= 0.75) & (freqs <= 3.0))
valid_freqs = freqs[interest_idx]
valid_mags = mag[interest_idx]
if len(valid_mags) == 0:
return None
max_idx = np.argmax(valid_mags)
peak_freq = valid_freqs[max_idx]
bpm = peak_freq * 60.0
# --- 5. 结果平滑 ---
# 防止数字乱跳
self.bpm_history.append(bpm)
avg_bpm = np.mean(self.bpm_history)
return int(avg_bpm)

172
reproject/analyzer.py
View File

@@ -3,17 +3,28 @@ import mediapipe as mp
import time
import numpy as np
from collections import deque
from geometry_utils import calculate_ear, calculate_mar_simple, estimate_head_pose, LEFT_EYE, RIGHT_EYE
from geometry_utils import (
calculate_ear,
calculate_mar_simple,
calculate_iris_pos,
estimate_head_pose,
LEFT_EYE,
RIGHT_EYE,
LEFT_EYE_GAZE_IDXS,
RIGHT_EYE_GAZE_IDXS,
)
from face_library import FaceLibrary
try:
from new_emotion_test import analyze_emotion_with_hsemotion
HAS_EMOTION_MODULE = True
except ImportError:
print("⚠️ 未找到 new_emotion_test.py，情绪功能将不可用")
HAS_EMOTION_MODULE = False
class MonitorSystem:
def __init__(self, face_db):
# 初始化 MediaPipe
@@ -22,32 +33,33 @@ class MonitorSystem:
max_num_faces=1,
refine_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5
min_tracking_confidence=0.5,
)
# 初始化人脸底库
self.face_lib = FaceLibrary(face_db)
# 状态变量
self.current_user = None
# --- 时间控制 ---
self.last_identity_check_time = 0
self.IDENTITY_CHECK_INTERVAL = 2.0
self.IDENTITY_CHECK_INTERVAL = 2.0
self.last_emotion_check_time = 0
self.EMOTION_CHECK_INTERVAL = 3.0
self.EMOTION_CHECK_INTERVAL = 3.0
# --- 历史数据 ---
self.HISTORY_LEN = 5
self.ear_history = deque(maxlen=self.HISTORY_LEN)
self.mar_history = deque(maxlen=self.HISTORY_LEN)
# 缓存上一次的检测结果
self.cached_emotion = {
"label": "detecting...",
"va": (0.0, 0.0)
}
self.iris_ratio_history = [
deque(maxlen=self.HISTORY_LEN),
deque(maxlen=self.HISTORY_LEN),
]
# 缓存上一次的检测结果
self.cached_emotion = {"label": "detecting...", "va": (0.0, 0.0)}
def _get_smoothed_value(self, history, current_val):
"""内部函数：计算滑动平均值"""
@@ -62,32 +74,37 @@ class MonitorSystem:
"""
h, w = frame.shape[:2]
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
results = self.face_mesh.process(rgb_frame)
analysis_data = {
"has_face": False,
"ear": 0.0,
"ear": 0.0,
"mar": 0.0,
"iris_ratio": (0.5, 0.5), # 0最左/上，1最右/下
"pose": (0, 0, 0),
"identity": self.current_user,
"emotion_label": self.cached_emotion["label"],
"emotion_va": self.cached_emotion["va"]
"emotion_va": self.cached_emotion["va"],
"landmark": (0, w, h, 0),
"frame": frame,
}
if not results.multi_face_landmarks:
self.ear_history.clear()
self.mar_history.clear()
self.iris_ratio_history[0].clear()
self.iris_ratio_history[1].clear()
return analysis_data
analysis_data["has_face"] = True
landmarks = results.multi_face_landmarks[0].landmark
# 计算 EAR
# 计算 EAR
left_ear = calculate_ear([landmarks[i] for i in LEFT_EYE], w, h)
right_ear = calculate_ear([landmarks[i] for i in RIGHT_EYE], w, h)
raw_ear = (left_ear + right_ear) / 2.0
# 计算 MAR
top = np.array([landmarks[13].x * w, landmarks[13].y * h])
bottom = np.array([landmarks[14].x * w, landmarks[14].y * h])
@@ -95,79 +112,136 @@ class MonitorSystem:
right = np.array([landmarks[308].x * w, landmarks[308].y * h])
raw_mar = calculate_mar_simple(top, bottom, left, right)
# 计算虹膜位置
left_iris_ratio = calculate_iris_pos(landmarks, LEFT_EYE_GAZE_IDXS, w, h)
right_iris_ratio = calculate_iris_pos(landmarks, RIGHT_EYE_GAZE_IDXS, w, h)
raw_iris_ratio = (
(left_iris_ratio[0] + right_iris_ratio[0]) / 2.0,
(left_iris_ratio[1] + right_iris_ratio[1]) / 2.0,
)
# --- 使用 History 进行数据平滑 ---
smoothed_ear = self._get_smoothed_value(self.ear_history, raw_ear)
smoothed_mar = self._get_smoothed_value(self.mar_history, raw_mar)
smoothed_iris_ratio = (
(self._get_smoothed_value(self.iris_ratio_history[0], raw_iris_ratio[0])),
(self._get_smoothed_value(self.iris_ratio_history[1], raw_iris_ratio[1])),
)
# 计算头部姿态
pitch, yaw, roll = estimate_head_pose(landmarks, w, h)
analysis_data.update({
"ear": round(smoothed_ear, 4),
"mar": round(smoothed_mar, 4),
"pose": (int(pitch), int(yaw), int(roll))
})
analysis_data.update(
{
"ear": round(smoothed_ear, 4),
"mar": round(smoothed_mar, 4),
"iris_ratio": (
round(smoothed_iris_ratio[0], 4),
round(smoothed_iris_ratio[1], 4),
),
"pose": (int(pitch), int(yaw), int(roll)),
}
)
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算人脸框
face_loc = (
int(min(ys) * h - 0.1 * h),
int(max(xs) * w + 0.1 * w),
int(max(ys) * h + 0.1 * h),
int(min(xs) * w - 0.1 * w),
)
pad = 30
face_loc = (
max(0, face_loc[0] - pad),
min(w, face_loc[1] + pad),
min(h, face_loc[2] + pad),
max(0, face_loc[3] - pad),
)
analysis_data["landmark"] = face_loc
# --- ROI处理(对比选择在哪里实现) ---
top = face_loc[0]
right = face_loc[1]
bottom = face_loc[2]
left = face_loc[3]
scale_factor = 10
small_bg = cv2.resize(
frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR
)
# 使用 INTER_NEAREST 马赛克效果
# 使用 INTER_LINEAR 毛玻璃模糊效果
blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
face_roi = frame[top:bottom, left:right]
blurred_frame[top:bottom, left:right] = face_roi
analysis_data["frame"] = blurred_frame
now = time.time()
# --- 身份识别 ---
if now - self.last_identity_check_time > self.IDENTITY_CHECK_INTERVAL:
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算人脸框
face_loc = (
int(min(ys) * h), int(max(xs) * w),
int(max(ys) * h), int(min(xs) * w)
)
pad = 20
face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad),
min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
# xs = [l.x for l in landmarks]
# ys = [l.y for l in landmarks]
# # 计算人脸框
# face_loc = (
# int(min(ys) * h), int(max(xs) * w),
# int(max(ys) * h), int(min(xs) * w)
# )
# pad = 20
# face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad),
# min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
match_result = self.face_lib.identify(rgb_frame, face_location=face_loc)
if match_result:
self.current_user = match_result["info"]
self.last_identity_check_time = now
analysis_data["identity"] = self.current_user
# --- 情绪识别 ---
if HAS_EMOTION_MODULE and (now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL):
if HAS_EMOTION_MODULE and (
now - self.last_emotion_check_time > self.EMOTION_CHECK_INTERVAL
):
if results.multi_face_landmarks:
landmarks = results.multi_face_landmarks[0].landmark
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算裁剪坐标
x_min = int(min(xs) * w)
x_max = int(max(xs) * w)
y_min = int(min(ys) * h)
y_max = int(max(ys) * h)
pad_x = int((x_max - x_min) * 0.2)
pad_y = int((y_max - y_min) * 0.2)
x_min = max(0, x_min - pad_x)
x_max = min(w, x_max + pad_x)
y_min = max(0, y_min - pad_y)
y_max = min(h, y_max + pad_y)
face_crop = frame[y_min:y_max, x_min:x_max]
if face_crop.size > 0:
try:
emo_results = analyze_emotion_with_hsemotion(face_crop)
if emo_results:
top_res = emo_results[0]
self.cached_emotion["label"] = top_res.get("emotion", "unknown")
self.cached_emotion["label"] = top_res.get(
"emotion", "unknown"
)
self.cached_emotion["va"] = top_res.get("vaVal", (0.0, 0.0))
except Exception as e:
print(f"情绪分析出错: {e}")
self.last_emotion_check_time = now
analysis_data["emotion_label"] = self.cached_emotion["label"]
analysis_data["emotion_va"] = self.cached_emotion["va"]
return analysis_data
return analysis_data

102
reproject/geometry_utils.py
View File

@@ -5,71 +5,115 @@ import cv2
LEFT_EYE = [33, 160, 158, 133, 153, 144]
# 右眼
RIGHT_EYE = [362, 385, 387, 263, 373, 380]
# 左眼虹膜关键点索引
LEFT_EYE_GAZE_IDXS = [33, 133, 159, 145, 468]
# 右眼虹膜关键点索引
RIGHT_EYE_GAZE_IDXS = [263, 362, 386, 374, 473]
# 嘴唇 (内圈)
LIPS = [78, 95, 88, 178, 87, 14, 317, 402, 318, 324, 308, 415, 310, 311, 312, 13]
def _euclidean_distance(point1, point2):
return np.linalg.norm(point1 - point2)
def calculate_ear(landmarks, width, height):
"""计算眼睛纵横比 EAR"""
# 坐标转换
points = np.array([(p.x * width, p.y * height) for p in landmarks])
# 垂直距离
v1 = _euclidean_distance(points[1], points[5])
v2 = _euclidean_distance(points[2], points[4])
# 水平距离
h = _euclidean_distance(points[0], points[3])
ear = (v1 + v2) / (2.0 * h)
return ear
def calculate_iris_pos(landmarks, indices, width, height):
p_left = np.array([landmarks[indices[0]].x * width, landmarks[indices[0]].y * height])
p_right = np.array([landmarks[indices[1]].x * width, landmarks[indices[1]].y * height])
p_top = np.array([landmarks[indices[2]].x * width, landmarks[indices[2]].y * height])
p_bottom = np.array([landmarks[indices[3]].x * width, landmarks[indices[3]].y * height])
p_iris = np.array([landmarks[indices[4]].x * width, landmarks[indices[4]].y * height])
# 修改为欧几里得距离计算
eye_width = _euclidean_distance(p_left, p_right)
iris_left = _euclidean_distance(p_iris, p_left)
if eye_width < 1e-3:
raw_x = 0.5
else :
raw_x = iris_left / eye_width
eye_height = _euclidean_distance(p_bottom, p_top)
iris_top = _euclidean_distance(p_iris, p_top)
if eye_height < 1e-3:
raw_y = 0.5
else:
raw_y = iris_top / eye_height
# x_min = 0.4
# x_max = 0.6
# ratio_x = abs(raw_x - x_min) / (x_max - x_min)
# y_min = 0.2
# y_max = 0.8
# ratio_y = abs(raw_y - y_min) / (y_max - y_min)
ratio_x = raw_x
ratio_y = raw_y
return max(0.0, min(1.0, ratio_x)), max(0.0, min(1.0, ratio_y))
def calculate_mar(landmarks, width, height):
"""计算嘴巴纵横比 MAR"""
points = np.array([(p.x * width, p.y * height) for p in landmarks])
pass
pass
def calculate_mar_simple(top, bottom, left, right):
h = _euclidean_distance(top, bottom)
w = _euclidean_distance(left, right)
return h / w
# geometry_utils.py 中的 estimate_head_pose 函数替换为以下内容
def estimate_head_pose(landmarks, width, height):
"""
计算头部姿态 (Pitch, Yaw, Roll)
返回单位：角度 (Degree)
"""
# 3D 模型点 (标准人脸模型)
model_points = np.array([
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corner
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0) # Right mouth corner
])
model_points = np.array(
[
(0.0, 0.0, 0.0), # Nose tip
(0.0, -330.0, -65.0), # Chin
(-225.0, 170.0, -135.0), # Left eye left corner
(225.0, 170.0, -135.0), # Right eye right corner
(-150.0, -150.0, -125.0), # Left Mouth corner
(150.0, -150.0, -125.0), # Right mouth corner
]
)
# MediaPipe 对应的关键点索引
idx_list = [1, 152, 33, 263, 61, 291]
image_points = []
for idx in idx_list:
p = landmarks[idx]
image_points.append((p.x * width, p.y * height))
image_points = np.array(image_points, dtype="double")
focal_length = width
center = (width / 2, height / 2)
camera_matrix = np.array(
[[focal_length, 0, center[0]],
[0, focal_length, center[1]],
[0, 0, 1]], dtype="double"
[[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]],
dtype="double",
)
dist_coeffs = np.zeros((4, 1))
dist_coeffs = np.zeros((4, 1))
# 求解PnP
success, rotation_vector, translation_vector = cv2.solvePnP(
@@ -80,15 +124,21 @@ def estimate_head_pose(landmarks, width, height):
angles, mtxR, mtxQ, Qx, Qy, Qz = cv2.RQDecomp3x3(rmat)
pitch = angles[0]
yaw = angles[1]
roll = angles[2]
yaw = angles[1]
roll = angles[2]
if pitch < -180: pitch += 360
if pitch > 180: pitch -= 360
if pitch < -180:
pitch += 360
if pitch > 180:
pitch -= 360
pitch = 180 - pitch if pitch > 0 else -pitch - 180
if yaw < -180: yaw += 360
if yaw > 180: yaw -= 360
if roll < -180: roll += 360
if roll > 180: roll -= 360
if yaw < -180:
yaw += 360
if yaw > 180:
yaw -= 360
if roll < -180:
roll += 360
if roll > 180:
roll -= 360
return pitch, yaw, roll
return pitch, yaw, roll

449
reproject/main.py
View File

@@ -1,3 +1,4 @@
from calendar import c
import cv2
import threading
import time
@@ -5,12 +6,16 @@ import queue
import socket
import json
import urllib.request
import struct
import struct
import numpy as np
import mediapipe as mp
from analyzer import MonitorSystem
from webrtc_server import WebRTCServer
from HeartRateMonitor import HeartRateMonitor
SERVER_HOST = '10.128.50.6'
SERVER_PORT = 65432
API_URL = "http://10.128.50.6:5000/api/states"
SERVER_HOST = "10.128.50.6"
SERVER_PORT = 65432
API_URL = "http://10.128.50.6:5000/api/states"
CAMERA_ID = "23373333"
BASIC_FACE_DB = {
@@ -18,32 +23,89 @@ BASIC_FACE_DB = {
"Yaoyu": {"name": "Yaoyu Zhang", "age": 20, "image-path": "yaoyu.jpg"},
}
mp_face_mesh_main = mp.solutions.face_mesh
face_mesh_main = mp_face_mesh_main.FaceMesh(
max_num_faces=1,
refine_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5,
)
frame_queue = queue.Queue(maxsize=2)
video_queue = queue.Queue(maxsize=1)
def apply_soft_roi(frame):
h, w = frame.shape[:2]
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
black_bg = np.zeros((h, w, 3), dtype=np.uint8)
data_queue = queue.Queue(maxsize=10)
results = face_mesh_main.process(rgb_frame)
if not results.multi_face_landmarks:
return frame
landmarks = results.multi_face_landmarks[0].landmark
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算人脸框
face_loc = (
int(min(ys) * h - 0.1 * h),
int(max(xs) * w + 0.1 * w),
int(max(ys) * h + 0.1 * h),
int(min(xs) * w - 0.1 * w),
)
pad = 30
face_loc = (
max(0, face_loc[0] - pad),
min(w, face_loc[1] + pad),
min(h, face_loc[2] + pad),
max(0, face_loc[3] - pad),
)
top = face_loc[0]
right = face_loc[1]
bottom = face_loc[2]
left = face_loc[3]
scale_factor = 10
small_bg = cv2.resize(
frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR
)
# 使用 INTER_NEAREST 马赛克效果
# 使用 INTER_LINEAR 毛玻璃模糊效果
blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
face_roi = frame[top:bottom, left:right]
blurred_frame[top:bottom, left:right] = face_roi
black_bg[top:bottom, left:right] = face_roi
return blurred_frame
frame_queue = queue.Queue(maxsize=2)
video_queue = queue.Queue(maxsize=10)
data_queue = queue.Queue(maxsize=10)
show_queue = queue.Queue(maxsize=10)
stop_event = threading.Event()
def capture_thread():
"""
采集线程：优化了分发逻辑，对视频流进行降频处理
"""
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
print("[Capture] 摄像头启动...")
frame_count = 0
last_time = time.time()
while not stop_event.is_set():
ret, frame = cap.read()
if not ret:
break
if not frame_queue.full():
frame_queue.put(frame)
else:
@@ -53,21 +115,27 @@ def capture_thread():
except queue.Empty:
pass
if frame_count % 2 == 0:
try:
if video_queue.full():
video_queue.get_nowait()
video_queue.put(frame)
except:
pass
# try:
# if video_queue.full():
# video_queue.get_nowait()
# video_queue.put(frame)
# except:
# pass
frame_count += 1
time.sleep(0.01)
# time.sleep(1 / 30)
# current_time = time.time()
# if current_time - last_time >= 1.0:
# print(f"[Capture] FPS: {frame_count}")
# frame_count = 0
# last_time = current_time
# print(current_time - last_time)
# last_time = current_time
cap.release()
print("[Capture] 线程结束")
def analysis_thread():
"""
核心分析线程：
@@ -76,15 +144,27 @@ def analysis_thread():
"""
monitor = MonitorSystem(BASIC_FACE_DB)
print("[Analysis] 分析系统启动...")
freq = 0
gap = 60
status = 0 # 0:open 1:close
last_time = time.time()
last_freq = 0
# heart_monitor = HeartRateMonitor()
while not stop_event.is_set():
try:
frame = frame_queue.get(timeout=1)
except queue.Empty:
continue
# 核心分析
result = monitor.process_frame(frame)
result["eye_close_freq"] = 0
result["heart_rate_bpm"] = 0
if video_queue.full():
video_queue.get_nowait()
video_queue.put(result["frame"])
payload = {
"id": CAMERA_ID,
@@ -92,91 +172,159 @@ def analysis_thread():
"name": "",
"ear": "",
"mar": "",
"iris_ratio": "",
"eye_close_freq": "",
"pose": "",
"emo_label": "",
"emo_va": ""
"emo_va": "",
"heart_rate_bpm": "",
}
if result["has_face"] and result["identity"]:
payload.update({
"name": result["identity"]["name"],
"ear": result["ear"],
"mar": result["mar"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"]
})
payload.update(
{
"name": result["identity"]["name"],
"ear": result["ear"],
"mar": result["mar"],
"iris_ratio": result["iris_ratio"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"],
}
)
elif result["has_face"]:
payload.update({
"name": "Unknown",
"ear": result["ear"],
"mar": result["mar"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"]
})
payload.update(
{
"name": "Unknown",
"ear": result["ear"],
"mar": result["mar"],
"iris_ratio": result["iris_ratio"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"],
}
)
if result["has_face"] and result["ear"] < 0.2:
if status == 0:
freq += 1
status = 1
elif result["has_face"] and result["ear"] >= 0.2:
if status == 1:
freq += 1
status = 0
if time.time() - last_time >= gap:
last_freq = freq / 2
freq = 0
last_time = time.time()
result["eye_close_freq"] = last_freq
payload["eye_close_freq"] = last_freq
# bpm = heart_monitor.process_frame(frame, result["landmark"])
# if bpm != None:
# result["heart_rate_bpm"] = bpm
# payload["heart_rate_bpm"] = bpm
if data_queue.full():
try:
_ = data_queue.get_nowait()
_ = data_queue.get_nowait()
except queue.Empty:
pass
data_queue.put(payload)
draw_debug_info(frame, result)
cv2.imshow("Monitor Client", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
stop_event.set()
show_queue.put((result["frame"], result))
# draw_debug_info(frame, result)
# cv2.imshow("Monitor Client", frame)
cv2.destroyAllWindows()
print("[Analysis] 分析线程结束")
def video_stream_thread():
"""
发送线程：优化了 Socket 设置和压缩参数
"""
print(f"[Video] 准备连接服务器 {SERVER_HOST}:{SERVER_PORT} ...")
server = WebRTCServer(fps=60)
server.start()
fourcc = cv2.VideoWriter_fourcc(*'avc1')
# jetson-nvenc 编码器
# bitrate = 1000000 # 1 Mbps
# gst_pipeline = (
# f"appsrc ! "
# f"video/x-raw, format=BGR ! "
# f"queue ! "
# f"videoconvert ! "
# f"video/x-raw, format=RGBA ! "
# f"nvvidconv ! "
# f"nvv4l2h264enc bitrate={bitrate} control-rate=1 profile=High ! "
# f"h264parse ! "
# f"qtmux ! "
# f"filesink location={filename} "
# )
# out = cv2.VideoWriter(gst_pipeline, cv2.CAP_GSTREAMER, 0, fps, (width, height))
out1 = cv2.VideoWriter('output1.mp4', fourcc, 30.0, (1280, 720))
out2 = cv2.VideoWriter('output2.mp4', fourcc, 30.0, (1280, 720))
while not stop_event.is_set():
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
s.connect((SERVER_HOST, SERVER_PORT))
print(f"[Video] 已连接")
camera_id_bytes = CAMERA_ID.encode('utf-8')
while not stop_event.is_set():
try:
frame = video_queue.get(timeout=1)
small_frame = cv2.resize(frame, (320, 240))
ret, buffer = cv2.imencode('.jpg', small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50])
if not ret: continue
frame_bytes = buffer.tobytes()
header_id_len = len(camera_id_bytes).to_bytes(4, 'big')
header_frame_len = len(frame_bytes).to_bytes(4, 'big')
packet = header_id_len + camera_id_bytes + header_frame_len + frame_bytes
s.sendall(packet)
except queue.Empty:
continue
except Exception as e:
print(f"[Video] 发送断开: {e}")
break
frame = video_queue.get(timeout=1)
# small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
server.provide_frame(frame)
out1.write(frame)
out2.write(frame)
except queue.Empty:
continue
except Exception as e:
print(f"[Video] 重连中... {e}")
time.sleep(3)
print(f"[Video] 发送错误: {e}")
continue
# while not stop_event.is_set():
# try:
# with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
# s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# s.connect((SERVER_HOST, SERVER_PORT))
# print(f"[Video] 已连接")
# camera_id_bytes = CAMERA_ID.encode("utf-8")
# while not stop_event.is_set():
# try:
# frame = video_queue.get(timeout=1)
# small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
# ret, buffer = cv2.imencode(
# ".jpg", small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50]
# )
# if not ret:
# continue
# frame_bytes = buffer.tobytes()
# header_id_len = len(camera_id_bytes).to_bytes(4, "big")
# header_frame_len = len(frame_bytes).to_bytes(4, "big")
# packet = (
# header_id_len
# + camera_id_bytes
# + header_frame_len
# + frame_bytes
# )
# s.sendall(packet)
# except queue.Empty:
# continue
# except Exception as e:
# print(f"[Video] 发送断开: {e}")
# break
# except Exception as e:
# print(f"[Video] 重连中... {e}")
# time.sleep(3)
out1.release()
out2.release()
print("[Video] 线程结束")
def data_upload_thread():
"""
周期性爆发模式
@@ -185,9 +333,9 @@ def data_upload_thread():
"""
print("[Data] 数据上报线程启动 (周期模式: 休眠30s -> 连发5次)")
LONG_SLEEP = 30
BURST_COUNT = 5
BURST_GAP = 1
LONG_SLEEP = 30
BURST_COUNT = 5
BURST_GAP = 1
while not stop_event.is_set():
# --- 阶段 1: 长休眠 (30秒) ---
@@ -196,7 +344,7 @@ def data_upload_thread():
# --- 阶段 2: 爆发发送 (5次) ---
print(f"[Data] 开始上报周期 (连发 {BURST_COUNT} 次)...")
try:
while not data_queue.empty():
data_queue.get_nowait()
@@ -210,63 +358,126 @@ def data_upload_thread():
break
try:
data = data_queue.get(timeout=1.5)
data = data_queue.get(timeout=1.5)
try:
req = urllib.request.Request(
url=API_URL,
data=json.dumps(data).encode('utf-8'),
headers={'Content-Type': 'application/json'},
method='POST'
data=json.dumps(data).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=2) as resp:
pass
# 打印日志
name_info = data['name'] if data['name'] else "NO-FACE"
print(f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}")
name_info = data["name"] if data["name"] else "NO-FACE"
print(
f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}"
)
except Exception as e:
print(f"[Data] Upload Error: {e}")
except queue.Empty:
print(f"[Data] 队列为空，跳过第 {i+1} 次发送")
if i < BURST_COUNT - 1:
stop_event.wait(BURST_GAP)
print("[Data] 数据上报线程结束")
def draw_debug_info(frame, result):
"""在画面上画出即时数据"""
if not result["has_face"]:
cv2.putText(frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(
frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
)
return
# 显示身份
id_text = result["identity"]["name"] if result["identity"] else "Unknown"
color = (0, 255, 0) if result["identity"] else (0, 255, 255)
cv2.putText(frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2)
cv2.putText(
frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2
)
# 显示数据
cv2.putText(frame, f"EAR: {result['ear']}", (20, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
cv2.putText(frame, f"MAR: {result['mar']}", (20, 95), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
if result['ear'] < 0.15:
cv2.putText(frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(
frame,
f"EAR: {result['ear']}",
(20, 70),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"MAR: {result['mar']}",
(20, 95),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"Iris Ratio: {result['iris_ratio']}",
(20, 190),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"Eye Close Freq: {result['eye_close_freq']}",
(20, 170),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 0, 255),
1,
)
cv2.putText(
frame,
f"Heart Rate BPM: {result['heart_rate_bpm']}",
(20, 210),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 165, 255),
1,
)
if result["ear"] < 0.15:
cv2.putText(
frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
)
p, y, r = result["pose"]
cv2.putText(frame, f"Pose: P{p} Y{y} R{r}", (20, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 1)
cv2.putText(
frame,
f"Pose: P{p} Y{y} R{r}",
(20, 120),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 255, 255),
1,
)
emo = result.get("emotion_label", "N/A")
va = result.get("emotion_va", (0,0))
va = result.get("emotion_va", (0, 0))
# 显示格式: Emo: happy (-0.5, 0.2)
emo_text = f"Emo: {emo} ({va[0]:.2f}, {va[1]:.2f})"
cv2.putText(frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1)
cv2.putText(
frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1
)
if __name__ == "__main__":
t1 = threading.Thread(target=capture_thread, daemon=True)
t2 = threading.Thread(target=analysis_thread, daemon=True)
t1 = threading.Thread(target=capture_thread, daemon=True)
t2 = threading.Thread(target=analysis_thread, daemon=True)
t3 = threading.Thread(target=video_stream_thread, daemon=True)
t4 = threading.Thread(target=data_upload_thread, daemon=True)
t4 = threading.Thread(target=data_upload_thread, daemon=True)
t1.start()
t2.start()
@@ -275,7 +486,19 @@ if __name__ == "__main__":
try:
while not stop_event.is_set():
time.sleep(1)
try:
frame, result = show_queue.get(timeout=1)
except queue.Empty:
continue
# frame = apply_soft_roi(frame)
display_frame = frame.copy()
draw_debug_info(display_frame, result)
cv2.imshow("Monitor Client", display_frame)
if cv2.waitKey(1) & 0xFF == ord("q"):
stop_event.set()
# time.sleep(1)
cv2.destroyAllWindows()
except KeyboardInterrupt:
print("停止程序...")
stop_event.set()
@@ -283,4 +506,4 @@ if __name__ == "__main__":
t1.join()
t2.join()
t3.join()
t4.join()
t4.join()

49
reproject/server_demo.py Normal file
View File

@@ -0,0 +1,49 @@
import time
import numpy as np
import cv2
from webrtc_server import WebRTCServer
FPS = 60
server = WebRTCServer(fps=FPS)
server.start()
def run_ani():
width, height = 1920, 1080
frame_count = 60
while True:
frame = np.zeros((height, width, 3), dtype=np.uint8)
center_x = (frame_count * 10) % width
cv2.circle(frame, (center_x, height // 2), 50, (0, 255, 0), -1)
cv2.putText(
frame,
f"AniCam Frame: {frame_count}",
(10, 30),
cv2.FONT_HERSHEY_SIMPLEX,
1,
(255, 255, 255),
2,
)
color = (frame_count * 5 % 256, 100, 200)
cv2.rectangle(frame, (50, 50), (150, 150), color, -1)
server.provide_frame(frame)
frame_count += 1
time.sleep(1 / FPS)
def run_cam(device_id):
cap = cv2.VideoCapture(device_id)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 2560)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 1440)
while True:
ret, frame = cap.read()
if ret:
server.provide_frame(frame)
cv2.imshow("Camera", frame)
time.sleep(1 / FPS)
if __name__ == "__main__":
run_cam(0)

97
reproject/test.py Normal file
View File

@@ -0,0 +1,97 @@
import cv2
import threading
import time
import queue
import socket
import json
import urllib.request
import struct
import mediapipe as mp
import numpy as np
from analyzer import MonitorSystem
mp_face_mesh = mp.solutions.face_mesh
face_mesh = mp_face_mesh.FaceMesh(
max_num_faces=1,
refine_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5
)
def apply_soft_roi(frame):
h, w = frame.shape[:2]
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
black_bg = np.zeros((h, w, 3), dtype=np.uint8)
results = face_mesh.process(rgb_frame)
if not results.multi_face_landmarks:
return frame
landmarks = results.multi_face_landmarks[0].landmark
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算人脸框
face_loc = (
int(min(ys) * h - 0.1 * h), int(max(xs) * w + 0.1 * w),
int(max(ys) * h + 0.1 * h), int(min(xs) * w - 0.1 * w)
)
pad = 30
face_loc = (max(0, face_loc[0]-pad), min(w, face_loc[1]+pad),
min(h, face_loc[2]+pad), max(0, face_loc[3]-pad))
top = face_loc[0]
right = face_loc[1]
bottom = face_loc[2]
left = face_loc[3]
scale_factor = 10
small_bg = cv2.resize(frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR)
# 使用 INTER_NEAREST 马赛克效果
# 使用 INTER_LINEAR 毛玻璃模糊效果
blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
face_roi = frame[top:bottom, left:right]
blurred_frame[top:bottom, left:right] = face_roi
black_bg[top:bottom, left:right] = face_roi
return blurred_frame
def test_compression_efficiency():
# 1. 读取一张测试图 (或者用摄像头抓一帧)
cap = cv2.VideoCapture(0)
time.sleep(5) # 等待摄像头稳定
while True:
ret, frame = cap.read()
cap.release()
if not ret:
print("无法读取摄像头")
return
# 2. 生成模糊处理后的图
processed_frame = apply_soft_roi(frame) # 使用你的函数
# 3. 【关键】模拟网络传输/存储：进行 JPG 编码
# 这里的 80 代表 JPEG 质量，模拟视频编码过程
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 80]
# 编码原图
_, encoded_original = cv2.imencode('.jpg', frame, encode_param)
original_size = len(encoded_original)
# 编码处理后的图
_, encoded_processed = cv2.imencode('.jpg', processed_frame, encode_param)
processed_size = len(encoded_processed)
# 4. 对比结果
print(f"原始画面编码后大小: {original_size / 1024:.2f} KB")
print(f"ROI处理编码后大小: {processed_size / 1024:.2f} KB")
savings = (original_size - processed_size) / original_size * 100
print(f"📉 带宽/存储节省了: {savings:.2f}%")
# 可视化对比
cv2.imshow("Original", frame)
cv2.imshow("Processed (Bandwidth Saver)", processed_frame)
# 运行测试
# 注意：你需要先定义 face_mesh 和 apply_soft_roi 才能运行
test_compression_efficiency()

157
reproject/webrtc_server.py Normal file
View File

@@ -0,0 +1,157 @@
import threading
import time
import asyncio
import socketio
import numpy as np
import cv2
import av
from aiortc import (
MediaStreamError,
RTCConfiguration,
RTCPeerConnection,
RTCSessionDescription,
VideoStreamTrack,
)
from aiortc.mediastreams import VIDEO_CLOCK_RATE, VIDEO_TIME_BASE
class WebRTCServer:
def __init__(self, fps):
self.pcs = set()
self.fps = fps
self.frameContainer = [None]
self.background_loop = asyncio.new_event_loop()
self._rtc_thread = threading.Thread(
target=self._start_background_loop,
args=(self.background_loop,),
daemon=True,
)
self._rtc_thread.start()
self.server = "ws://10.128.50.6:5000"
def _start_background_loop(self, loop):
asyncio.set_event_loop(loop)
loop.run_forever()
async def _websocket_start(self):
sio = socketio.AsyncClient()
@sio.event
async def connect():
print("已连接到中心信令服务器")
# TODO 注册自己为设备
await sio.emit("checkin", {"device_id": "cam_001"})
@sio.event
async def offer(data):
# data 包含: { sdp: "...", type: "offer", "sid": "..." }
print("收到 WebRTC Offer")
localDescription = await self._handle_offer(data)
await sio.emit(
"answer",
{
"sdp": localDescription.sdp,
"type": localDescription.type,
"sid": data["sid"],
},
)
await sio.connect(self.server)
print(self.server, "connected")
await sio.wait()
async def _handle_offer(self, offer):
pc = RTCPeerConnection(RTCConfiguration(iceServers=[]))
self.pcs.add(pc)
start = time.time()
@pc.on("connectionstatechange")
async def on_state_change():
if pc.connectionState == "failed" or pc.connectionState == "closed":
await pc.close()
print("Connection closed")
self.pcs.discard(pc)
@pc.on("datachannel")
def on_data_channel(channel):
route_channel(channel)
await pc.setRemoteDescription(
RTCSessionDescription(offer["sdp"], offer.get("type", "offer"))
)
pc.addTrack(VideoFrameTrack(self.fps, self.frameContainer))
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
print(f"Handle offer in {(time.time() - start)*1000:.2f}ms")
return pc.localDescription
def start(self):
asyncio.run_coroutine_threadsafe(self._websocket_start(), self.background_loop)
def provide_frame(self, frame):
self.frameContainer[0] = frame
def stop(self):
if self.background_loop.is_running():
self.background_loop.call_soon_threadsafe(self.background_loop.stop)
if self._rtc_thread.is_alive():
self._rtc_thread.join(timeout=2)
class VideoFrameTrack(VideoStreamTrack):
def __init__(self, fps, fc):
super().__init__()
self.fps = fps
self.frameContainer = fc
async def next_timestamp(self):
"""
重写父类方法，去除帧率限制
"""
if self.readyState != "live":
raise MediaStreamError
if hasattr(self, "_timestamp"):
self._timestamp += int(1 / self.fps * VIDEO_CLOCK_RATE)
wait = self._start + (self._timestamp / VIDEO_CLOCK_RATE) - time.time()
await asyncio.sleep(wait)
else:
self._start = time.time()
self._timestamp = 0
return self._timestamp, VIDEO_TIME_BASE
async def recv(self):
pts, time_base = await self.next_timestamp()
frame = self.frameContainer[0]
if frame is None:
frame = np.zeros((480, 640, 3), dtype=np.uint8)
else:
frame = self.frameContainer[0]
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
video_frame = av.VideoFrame.from_ndarray(frame, format="rgb24")
video_frame.pts = pts
video_frame.time_base = time_base
return video_frame
def route_channel(channel):
match channel.label:
case "latency":
@channel.on("message")
def on_message(message):
now = int(time.time() * 1000 + 0.5)
channel.send(str(now))
pre = int(message)
print(f"Latency: {now - pre}ms")
case _:
print(f"Unknown Channel {channel.label}")