colden/cv_state_ana
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

眨眼频率、眼动分析、心率、视频录制

Browse Source
This commit is contained in:
邓智航
2026-01-25 15:19:01 +08:00
parent 6e882d2aa4
commit b3997c2646
7 changed files with 977 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

449
reproject/main.py
View File

@@ -1,3 +1,4 @@
from calendar import c
import cv2
import threading
import time
@@ -5,12 +6,16 @@ import queue
import socket
import json
import urllib.request
import struct
import struct
import numpy as np
import mediapipe as mp
from analyzer import MonitorSystem
from webrtc_server import WebRTCServer
from HeartRateMonitor import HeartRateMonitor
SERVER_HOST = '10.128.50.6'
SERVER_PORT = 65432
API_URL = "http://10.128.50.6:5000/api/states"
SERVER_HOST = "10.128.50.6"
SERVER_PORT = 65432
API_URL = "http://10.128.50.6:5000/api/states"
CAMERA_ID = "23373333"
BASIC_FACE_DB = {
@@ -18,32 +23,89 @@ BASIC_FACE_DB = {
"Yaoyu": {"name": "Yaoyu Zhang", "age": 20, "image-path": "yaoyu.jpg"},
}
mp_face_mesh_main = mp.solutions.face_mesh
face_mesh_main = mp_face_mesh_main.FaceMesh(
max_num_faces=1,
refine_landmarks=True,
min_detection_confidence=0.5,
min_tracking_confidence=0.5,
)
frame_queue = queue.Queue(maxsize=2)
video_queue = queue.Queue(maxsize=1)
def apply_soft_roi(frame):
h, w = frame.shape[:2]
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
black_bg = np.zeros((h, w, 3), dtype=np.uint8)
data_queue = queue.Queue(maxsize=10)
results = face_mesh_main.process(rgb_frame)
if not results.multi_face_landmarks:
return frame
landmarks = results.multi_face_landmarks[0].landmark
xs = [l.x for l in landmarks]
ys = [l.y for l in landmarks]
# 计算人脸框
face_loc = (
int(min(ys) * h - 0.1 * h),
int(max(xs) * w + 0.1 * w),
int(max(ys) * h + 0.1 * h),
int(min(xs) * w - 0.1 * w),
)
pad = 30
face_loc = (
max(0, face_loc[0] - pad),
min(w, face_loc[1] + pad),
min(h, face_loc[2] + pad),
max(0, face_loc[3] - pad),
)
top = face_loc[0]
right = face_loc[1]
bottom = face_loc[2]
left = face_loc[3]
scale_factor = 10
small_bg = cv2.resize(
frame, (w // scale_factor, h // scale_factor), interpolation=cv2.INTER_LINEAR
)
# 使用 INTER_NEAREST 马赛克效果
# 使用 INTER_LINEAR 毛玻璃模糊效果
blurred_frame = cv2.resize(small_bg, (w, h), interpolation=cv2.INTER_LINEAR)
face_roi = frame[top:bottom, left:right]
blurred_frame[top:bottom, left:right] = face_roi
black_bg[top:bottom, left:right] = face_roi
return blurred_frame
frame_queue = queue.Queue(maxsize=2)
video_queue = queue.Queue(maxsize=10)
data_queue = queue.Queue(maxsize=10)
show_queue = queue.Queue(maxsize=10)
stop_event = threading.Event()
def capture_thread():
"""
采集线程：优化了分发逻辑，对视频流进行降频处理
"""
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, 1280)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 720)
print("[Capture] 摄像头启动...")
frame_count = 0
last_time = time.time()
while not stop_event.is_set():
ret, frame = cap.read()
if not ret:
break
if not frame_queue.full():
frame_queue.put(frame)
else:
@@ -53,21 +115,27 @@ def capture_thread():
except queue.Empty:
pass
if frame_count % 2 == 0:
try:
if video_queue.full():
video_queue.get_nowait()
video_queue.put(frame)
except:
pass
# try:
# if video_queue.full():
# video_queue.get_nowait()
# video_queue.put(frame)
# except:
# pass
frame_count += 1
time.sleep(0.01)
# time.sleep(1 / 30)
# current_time = time.time()
# if current_time - last_time >= 1.0:
# print(f"[Capture] FPS: {frame_count}")
# frame_count = 0
# last_time = current_time
# print(current_time - last_time)
# last_time = current_time
cap.release()
print("[Capture] 线程结束")
def analysis_thread():
"""
核心分析线程：
@@ -76,15 +144,27 @@ def analysis_thread():
"""
monitor = MonitorSystem(BASIC_FACE_DB)
print("[Analysis] 分析系统启动...")
freq = 0
gap = 60
status = 0 # 0:open 1:close
last_time = time.time()
last_freq = 0
# heart_monitor = HeartRateMonitor()
while not stop_event.is_set():
try:
frame = frame_queue.get(timeout=1)
except queue.Empty:
continue
# 核心分析
result = monitor.process_frame(frame)
result["eye_close_freq"] = 0
result["heart_rate_bpm"] = 0
if video_queue.full():
video_queue.get_nowait()
video_queue.put(result["frame"])
payload = {
"id": CAMERA_ID,
@@ -92,91 +172,159 @@ def analysis_thread():
"name": "",
"ear": "",
"mar": "",
"iris_ratio": "",
"eye_close_freq": "",
"pose": "",
"emo_label": "",
"emo_va": ""
"emo_va": "",
"heart_rate_bpm": "",
}
if result["has_face"] and result["identity"]:
payload.update({
"name": result["identity"]["name"],
"ear": result["ear"],
"mar": result["mar"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"]
})
payload.update(
{
"name": result["identity"]["name"],
"ear": result["ear"],
"mar": result["mar"],
"iris_ratio": result["iris_ratio"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"],
}
)
elif result["has_face"]:
payload.update({
"name": "Unknown",
"ear": result["ear"],
"mar": result["mar"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"]
})
payload.update(
{
"name": "Unknown",
"ear": result["ear"],
"mar": result["mar"],
"iris_ratio": result["iris_ratio"],
"pose": result["pose"],
"emo_label": result["emotion_label"],
"emo_va": result["emotion_va"],
}
)
if result["has_face"] and result["ear"] < 0.2:
if status == 0:
freq += 1
status = 1
elif result["has_face"] and result["ear"] >= 0.2:
if status == 1:
freq += 1
status = 0
if time.time() - last_time >= gap:
last_freq = freq / 2
freq = 0
last_time = time.time()
result["eye_close_freq"] = last_freq
payload["eye_close_freq"] = last_freq
# bpm = heart_monitor.process_frame(frame, result["landmark"])
# if bpm != None:
# result["heart_rate_bpm"] = bpm
# payload["heart_rate_bpm"] = bpm
if data_queue.full():
try:
_ = data_queue.get_nowait()
_ = data_queue.get_nowait()
except queue.Empty:
pass
data_queue.put(payload)
draw_debug_info(frame, result)
cv2.imshow("Monitor Client", frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
stop_event.set()
show_queue.put((result["frame"], result))
# draw_debug_info(frame, result)
# cv2.imshow("Monitor Client", frame)
cv2.destroyAllWindows()
print("[Analysis] 分析线程结束")
def video_stream_thread():
"""
发送线程：优化了 Socket 设置和压缩参数
"""
print(f"[Video] 准备连接服务器 {SERVER_HOST}:{SERVER_PORT} ...")
server = WebRTCServer(fps=60)
server.start()
fourcc = cv2.VideoWriter_fourcc(*'avc1')
# jetson-nvenc 编码器
# bitrate = 1000000 # 1 Mbps
# gst_pipeline = (
# f"appsrc ! "
# f"video/x-raw, format=BGR ! "
# f"queue ! "
# f"videoconvert ! "
# f"video/x-raw, format=RGBA ! "
# f"nvvidconv ! "
# f"nvv4l2h264enc bitrate={bitrate} control-rate=1 profile=High ! "
# f"h264parse ! "
# f"qtmux ! "
# f"filesink location={filename} "
# )
# out = cv2.VideoWriter(gst_pipeline, cv2.CAP_GSTREAMER, 0, fps, (width, height))
out1 = cv2.VideoWriter('output1.mp4', fourcc, 30.0, (1280, 720))
out2 = cv2.VideoWriter('output2.mp4', fourcc, 30.0, (1280, 720))
while not stop_event.is_set():
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
s.connect((SERVER_HOST, SERVER_PORT))
print(f"[Video] 已连接")
camera_id_bytes = CAMERA_ID.encode('utf-8')
while not stop_event.is_set():
try:
frame = video_queue.get(timeout=1)
small_frame = cv2.resize(frame, (320, 240))
ret, buffer = cv2.imencode('.jpg', small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50])
if not ret: continue
frame_bytes = buffer.tobytes()
header_id_len = len(camera_id_bytes).to_bytes(4, 'big')
header_frame_len = len(frame_bytes).to_bytes(4, 'big')
packet = header_id_len + camera_id_bytes + header_frame_len + frame_bytes
s.sendall(packet)
except queue.Empty:
continue
except Exception as e:
print(f"[Video] 发送断开: {e}")
break
frame = video_queue.get(timeout=1)
# small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
server.provide_frame(frame)
out1.write(frame)
out2.write(frame)
except queue.Empty:
continue
except Exception as e:
print(f"[Video] 重连中... {e}")
time.sleep(3)
print(f"[Video] 发送错误: {e}")
continue
# while not stop_event.is_set():
# try:
# with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
# s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
# s.connect((SERVER_HOST, SERVER_PORT))
# print(f"[Video] 已连接")
# camera_id_bytes = CAMERA_ID.encode("utf-8")
# while not stop_event.is_set():
# try:
# frame = video_queue.get(timeout=1)
# small_frame = cv2.resize(apply_soft_roi(frame), (1280, 720))
# ret, buffer = cv2.imencode(
# ".jpg", small_frame, [cv2.IMWRITE_JPEG_QUALITY, 50]
# )
# if not ret:
# continue
# frame_bytes = buffer.tobytes()
# header_id_len = len(camera_id_bytes).to_bytes(4, "big")
# header_frame_len = len(frame_bytes).to_bytes(4, "big")
# packet = (
# header_id_len
# + camera_id_bytes
# + header_frame_len
# + frame_bytes
# )
# s.sendall(packet)
# except queue.Empty:
# continue
# except Exception as e:
# print(f"[Video] 发送断开: {e}")
# break
# except Exception as e:
# print(f"[Video] 重连中... {e}")
# time.sleep(3)
out1.release()
out2.release()
print("[Video] 线程结束")
def data_upload_thread():
"""
周期性爆发模式
@@ -185,9 +333,9 @@ def data_upload_thread():
"""
print("[Data] 数据上报线程启动 (周期模式: 休眠30s -> 连发5次)")
LONG_SLEEP = 30
BURST_COUNT = 5
BURST_GAP = 1
LONG_SLEEP = 30
BURST_COUNT = 5
BURST_GAP = 1
while not stop_event.is_set():
# --- 阶段 1: 长休眠 (30秒) ---
@@ -196,7 +344,7 @@ def data_upload_thread():
# --- 阶段 2: 爆发发送 (5次) ---
print(f"[Data] 开始上报周期 (连发 {BURST_COUNT} 次)...")
try:
while not data_queue.empty():
data_queue.get_nowait()
@@ -210,63 +358,126 @@ def data_upload_thread():
break
try:
data = data_queue.get(timeout=1.5)
data = data_queue.get(timeout=1.5)
try:
req = urllib.request.Request(
url=API_URL,
data=json.dumps(data).encode('utf-8'),
headers={'Content-Type': 'application/json'},
method='POST'
data=json.dumps(data).encode("utf-8"),
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=2) as resp:
pass
# 打印日志
name_info = data['name'] if data['name'] else "NO-FACE"
print(f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}")
name_info = data["name"] if data["name"] else "NO-FACE"
print(
f"[Data Upload {i+1}/{BURST_COUNT}] {name_info} | Time:{data['time']}"
)
except Exception as e:
print(f"[Data] Upload Error: {e}")
except queue.Empty:
print(f"[Data] 队列为空，跳过第 {i+1} 次发送")
if i < BURST_COUNT - 1:
stop_event.wait(BURST_GAP)
print("[Data] 数据上报线程结束")
def draw_debug_info(frame, result):
"""在画面上画出即时数据"""
if not result["has_face"]:
cv2.putText(frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(
frame, "NO FACE", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
)
return
# 显示身份
id_text = result["identity"]["name"] if result["identity"] else "Unknown"
color = (0, 255, 0) if result["identity"] else (0, 255, 255)
cv2.putText(frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2)
cv2.putText(
frame, f"User: {id_text}", (20, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.7, color, 2
)
# 显示数据
cv2.putText(frame, f"EAR: {result['ear']}", (20, 70), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
cv2.putText(frame, f"MAR: {result['mar']}", (20, 95), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (255, 255, 0), 1)
if result['ear'] < 0.15:
cv2.putText(frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.putText(
frame,
f"EAR: {result['ear']}",
(20, 70),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"MAR: {result['mar']}",
(20, 95),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"Iris Ratio: {result['iris_ratio']}",
(20, 190),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 255, 0),
1,
)
cv2.putText(
frame,
f"Eye Close Freq: {result['eye_close_freq']}",
(20, 170),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(255, 0, 255),
1,
)
cv2.putText(
frame,
f"Heart Rate BPM: {result['heart_rate_bpm']}",
(20, 210),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 165, 255),
1,
)
if result["ear"] < 0.15:
cv2.putText(
frame, "EYE CLOSE", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2
)
p, y, r = result["pose"]
cv2.putText(frame, f"Pose: P{p} Y{y} R{r}", (20, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 255, 255), 1)
cv2.putText(
frame,
f"Pose: P{p} Y{y} R{r}",
(20, 120),
cv2.FONT_HERSHEY_SIMPLEX,
0.6,
(0, 255, 255),
1,
)
emo = result.get("emotion_label", "N/A")
va = result.get("emotion_va", (0,0))
va = result.get("emotion_va", (0, 0))
# 显示格式: Emo: happy (-0.5, 0.2)
emo_text = f"Emo: {emo} ({va[0]:.2f}, {va[1]:.2f})"
cv2.putText(frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1)
cv2.putText(
frame, emo_text, (20, 145), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 165, 255), 1
)
if __name__ == "__main__":
t1 = threading.Thread(target=capture_thread, daemon=True)
t2 = threading.Thread(target=analysis_thread, daemon=True)
t1 = threading.Thread(target=capture_thread, daemon=True)
t2 = threading.Thread(target=analysis_thread, daemon=True)
t3 = threading.Thread(target=video_stream_thread, daemon=True)
t4 = threading.Thread(target=data_upload_thread, daemon=True)
t4 = threading.Thread(target=data_upload_thread, daemon=True)
t1.start()
t2.start()
@@ -275,7 +486,19 @@ if __name__ == "__main__":
try:
while not stop_event.is_set():
time.sleep(1)
try:
frame, result = show_queue.get(timeout=1)
except queue.Empty:
continue
# frame = apply_soft_roi(frame)
display_frame = frame.copy()
draw_debug_info(display_frame, result)
cv2.imshow("Monitor Client", display_frame)
if cv2.waitKey(1) & 0xFF == ord("q"):
stop_event.set()
# time.sleep(1)
cv2.destroyAllWindows()
except KeyboardInterrupt:
print("停止程序...")
stop_event.set()
@@ -283,4 +506,4 @@ if __name__ == "__main__":
t1.join()
t2.join()
t3.join()
t4.join()
t4.join()