基于人脸68特征点的疲劳驾驶检测

疲劳驾驶检测"/>

基于人脸68特征点的疲劳驾驶检测

 本文代码在别人代码上添加了滤波和调整了眼部和嘴部的判断阈值

 基本配置Requirements.txt并安装：

pip install -r Requirements.txt

imutils==0.5.3
opencv-contrib-python==4.2.0.34
opencv-python==4.2.0.34
dlib==19.20.0
numpy==1.17.3
cmake==3.18.0
scipy==1.5.1

需要下载68特征点文件.dat文件

主检测代码如下Detect.py

#!/usr/bin/env python
from scipy.spatial import distance as dist
from imutils.video import VideoStream
from imutils import face_utils
import argparse
import imutils
import time
import dlib
import math
from cv2 import cv2
import numpy as np
from EAR import eye_aspect_ratio
from MAR import mouth_aspect_ratio
from HeadPose import getHeadTiltAndCoords# initialize dlib's face detector (HOG-based) and then create the
# facial landmark predictor
print("[INFO] loading facial landmark predictor...")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('./dlib_shape_predictor/shape_predictor_68_face_landmarks.dat')# initialize the video stream and sleep for a bit, allowing the
# camera sensor to warm up
print("[INFO] initializing camera...")vs = VideoStream(src=1).start()
# vs = VideoStream(usePiCamera=True).start() # Raspberry Pi
time.sleep(2.0)# 400x225 to 1024x576
frame_width = 1024
frame_height = 576# loop over the frames from the video stream
# 2D image points. If you change the image, you need to change vector
image_points = np.array([(359, 391),     # Nose tip 34(399, 561),     # Chin 9(337, 297),     # Left eye left corner 37(513, 301),     # Right eye right corne 46(345, 465),     # Left Mouth corner 49(453, 469)      # Right mouth corner 55
], dtype="double")(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]EYE_AR_THRESH = 0.16
MOUTH_AR_THRESH = 0.87
EYE_AR_CONSEC_FRAMES = 6
COUNTER = 0# grab the indexes of the facial landmarks for the mouth
(mStart, mEnd) = (49, 68)while True:# grab the frame from the threaded video stream, resize it to# have a maximum width of 400 pixels, and convert it to# grayscaleframe = vs.read()frame = imutils.resize(frame, width=1024, height=576)gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)gray = cv2.medianBlur(gray, 5)size = gray.shape# detect faces in the grayscale framerects = detector(gray, 0)# check to see if a face was detected, and if so, draw the total# number of faces on the frameif len(rects) > 0:text = "{} face(s) found".format(len(rects))cv2.putText(frame, text, (10, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)# loop over the face detectionsfor rect in rects:# compute the bounding box of the face and draw it on the# frame(bX, bY, bW, bH) = face_utils.rect_to_bb(rect)cv2.rectangle(frame, (bX, bY), (bX + bW, bY + bH), (0, 255, 0), 1)# determine the facial landmarks for the face region, then# convert the facial landmark (x, y)-coordinates to a NumPy# arrayshape = predictor(gray, rect)shape = face_utils.shape_to_np(shape)# extract the left and right eye coordinates, then use the# coordinates to compute the eye aspect ratio for both eyesleftEye = shape[lStart:lEnd]rightEye = shape[rStart:rEnd]leftEAR = eye_aspect_ratio(leftEye)rightEAR = eye_aspect_ratio(rightEye)# average the eye aspect ratio together for both eyesear = (leftEAR + rightEAR) / 2.0# compute the convex hull for the left and right eye, then# visualize each of the eyesleftEyeHull = cv2.convexHull(leftEye)rightEyeHull = cv2.convexHull(rightEye)cv2.drawContours(frame, [leftEyeHull], -1, (0, 255, 0), 1)cv2.drawContours(frame, [rightEyeHull], -1, (0, 255, 0), 1)# check to see if the eye aspect ratio is below the blink# threshold, and if so, increment the blink frame counterif ear < EYE_AR_THRESH:COUNTER += 1# if the eyes were closed for a sufficient number of times# then show the warningif COUNTER >= EYE_AR_CONSEC_FRAMES:cv2.putText(frame, "Eyes Closed!", (500, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)# otherwise, the eye aspect ratio is not below the blink# threshold, so reset the counter and alarmelse:COUNTER = 0mouth = shape[mStart:mEnd]mouthMAR = mouth_aspect_ratio(mouth)mar = mouthMAR# compute the convex hull for the mouth, then# visualize the mouthmouthHull = cv2.convexHull(mouth)cv2.drawContours(frame, [mouthHull], -1, (0, 255, 0), 1)cv2.putText(frame, "MAR: {:.2f}".format(mar), (650, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)# Draw text if mouth is openif mar > MOUTH_AR_THRESH:cv2.putText(frame, "Yawning!", (800, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)# loop over the (x, y)-coordinates for the facial landmarks# and draw each of themfor (i, (x, y)) in enumerate(shape):if i == 33:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[0] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)elif i == 8:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[1] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)elif i == 36:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[2] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)elif i == 45:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[3] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)elif i == 48:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[4] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)elif i == 54:# something to our key landmarks# save to our new key point list# i.e. keypoints = [(i,(x,y))]image_points[5] = np.array([x, y], dtype='double')# write on frame in Greencv2.circle(frame, (x, y), 1, (0, 255, 0), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 255, 0), 1)else:# everything to all other landmarks# write on frame in Redcv2.circle(frame, (x, y), 1, (0, 0, 255), -1)cv2.putText(frame, str(i + 1), (x - 10, y - 10),cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1)#Draw the determinant image points onto the person's facefor p in image_points:cv2.circle(frame, (int(p[0]), int(p[1])), 3, (0, 0, 255), -1)(head_tilt_degree, start_point, end_point, end_point_alt) = getHeadTiltAndCoords(size, image_points, frame_height)cv2.line(frame, start_point, end_point, (255, 0, 0), 2)cv2.line(frame, start_point, end_point_alt, (0, 0, 255), 2)if head_tilt_degree:cv2.putText(frame, 'Head Tilt Degree: ' + str(head_tilt_degree[0]), (170, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)# extract the mouth coordinates, then use the# coordinates to compute the mouth aspect ratio# show the frameqcv2.imshow("Frame", frame)key = cv2.waitKey(1) & 0xFF# if the `q` key was pressed, break from the loopif key == ord("q"):break# print(image_points)# do a bit of cleanup
cv2.destroyAllWindows()
vs.stop()

眼部EAR.py

from scipy.spatial import distance as distdef eye_aspect_ratio(eye):# compute the euclidean distances between the two sets of# vertical eye landmarks (x, y)-coordinatesA = dist.euclidean(eye[1], eye[5])B = dist.euclidean(eye[2], eye[4])# compute the euclidean distance between the horizontal# eye landmark (x, y)-coordinatesC = dist.euclidean(eye[0], eye[3])# compute the eye aspect ratioear = (A + B) / (2.0 * C)# return the eye aspect ratioreturn ear

嘴部MAR.py

from scipy.spatial import distance as distdef mouth_aspect_ratio(mouth):# compute the euclidean distances between the two sets of# vertical mouth landmarks (x, y)-coordinatesA = dist.euclidean(mouth[2], mouth[10])  # 51, 59B = dist.euclidean(mouth[4], mouth[8])  # 53, 57# compute the euclidean distance between the horizontal# mouth landmark (x, y)-coordinatesC = dist.euclidean(mouth[0], mouth[6])  # 49, 55# compute the mouth aspect ratiomar = (A + B) / (2.0 * C)# return the mouth aspect ratioreturn mar

头部 HeadPose.py

import numpy as np
import math
from cv2 import cv2# 3D model points.
model_points = np.array([(0.0, 0.0, 0.0),             # Nose tip 34(0.0, -330.0, -65.0),        # Chin 9(-225.0, 170.0, -135.0),     # Left eye left corner 37(225.0, 170.0, -135.0),      # Right eye right corne 46(-150.0, -150.0, -125.0),    # Left Mouth corner 49(150.0, -150.0, -125.0)      # Right mouth corner 55
])# Checks if a matrix is a valid rotation matrix.
def isRotationMatrix(R):Rt = np.transpose(R)shouldBeIdentity = np.dot(Rt, R)I = np.identity(3, dtype=R.dtype)n = np.linalg.norm(I - shouldBeIdentity)return n < 1e-6# Calculates rotation matrix to euler angles
# The result is the same as MATLAB except the order
# of the euler angles ( x and z are swapped ).
def rotationMatrixToEulerAngles(R):assert(isRotationMatrix(R))sy = math.sqrt(R[0, 0] * R[0, 0] + R[1, 0] * R[1, 0])singular = sy < 1e-6if not singular:x = math.atan2(R[2, 1], R[2, 2])y = math.atan2(-R[2, 0], sy)z = math.atan2(R[1, 0], R[0, 0])else:x = math.atan2(-R[1, 2], R[1, 1])y = math.atan2(-R[2, 0], sy)z = 0return np.array([x, y, z])def getHeadTiltAndCoords(size, image_points, frame_height):focal_length = size[1]center = (size[1]/2, size[0]/2)camera_matrix = np.array([[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]], dtype="double")# print "Camera Matrix :\n {0}".format(camera_matrix)dist_coeffs = np.zeros((4, 1))  # Assuming no lens distortion(_, rotation_vector, translation_vector) = cv2.solvePnP(model_points, image_points,camera_matrix, dist_coeffs, flags = cv2.SOLVEPNP_ITERATIVE)  # flags=cv2.CV_ITERATIVE)# print "Rotation Vector:\n {0}".format(rotation_vector)# print "Translation Vector:\n {0}".format(translation_vector)# Project a 3D point (0, 0 , 1000.0) onto the image plane# We use this to draw a line sticking out of the nose_end_point2D(nose_end_point2D, _) = cv2.projectPoints(np.array([(0.0, 0.0, 1000.0)]), rotation_vector, translation_vector, camera_matrix, dist_coeffs)#get rotation matrix from the rotation vectorrotation_matrix, _ = cv2.Rodrigues(rotation_vector)#calculate head tilt angle in degreeshead_tilt_degree = abs([-180] - np.rad2deg([rotationMatrixToEulerAngles(rotation_matrix)[0]]))#calculate starting and ending points for the two lines for illustrationstarting_point = (int(image_points[0][0]), int(image_points[0][1]))ending_point = (int(nose_end_point2D[0][0][0]), int(nose_end_point2D[0][0][1]))ending_point_alternate = (ending_point[0], frame_height // 2)return head_tilt_degree, starting_point, ending_point, ending_point_alternate

简单不显示检测的详细信息版

#!/usr/bin/env python
from scipy.spatial import distance as dist
from imutils.video import VideoStream
from imutils import face_utils
import argparse
import imutils
import time
import dlib
import math
from cv2 import cv2
import numpy as np
from EAR import eye_aspect_ratio
from MAR import mouth_aspect_ratio
from HeadPose import getHeadTiltAndCoords# initialize dlib's face detector (HOG-based) and then create the
# facial landmark predictor 
# 初始化dlib的人脸检测器(基于hog)，然后创建面部点检测器
print("[INFO] loading facial landmark predictor...")
detector = dlib.get_frontal_face_detector()
predictor = dlib.shape_predictor('./dlib_shape_predictor/shape_predictor_68_face_landmarks.dat')# initialize the video stream and sleep for a bit, allowing the
# camera sensor to warm up 
# 初始化视流并休眠一段时间，允许摄像机warm up
print("[INFO] initializing camera...")vs = VideoStream(src=0).start()
# vs = VideoStream(usePiCamera=True).start() # Raspberry Pi
time.sleep(2.0)# 400x225 to 1024x576
frame_width = 1024
frame_height = 576# loop over the frames from the video stream 
# 从视频流中循环显示视频帧。如果你改变图像，你需要改变向量
# 2D image points. If you change the image, you need to change vector
image_points = np.array([(359, 391),     # Nose tip 34(399, 561),     # Chin 9(337, 297),     # Left eye left corner 37(513, 301),     # Right eye right corne 46(345, 465),     # Left Mouth corner 49(453, 469)      # Right mouth corner 55
], dtype="double")(lStart, lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"]
(rStart, rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"]EYE_AR_THRESH = 0.16
MOUTH_AR_THRESH = 0.87
EYE_AR_CONSEC_FRAMES = 6
COUNTER = 0# grab the indexes of the facial landmarks for the mouth 
# 用面部特征点的索引来做嘴巴
(mStart, mEnd) = (49, 68)while True:# grab the frame from the threaded video stream, resize it to# have a maximum width of 400 pixels, and convert it to# grayscale# 从线程视频流中获取帧，将其调整为最大宽度为400像素，并将其转换为灰度frame = vs.read()frame = imutils.resize(frame, width=1024, height=576)gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)gray = cv2.medianBlur(gray, 5)size = gray.shape# detect faces in the grayscale frame# 在灰度图中检测人脸rects = detector(gray, 0)# check to see if a face was detected, and if so, draw the total# number of faces on the frame # 检查是否检测到一张脸，如果检测到，在帧上画出脸的总数if len(rects) > 0:text = "{} faces found".format(len(rects))cv2.putText(frame, text, (10, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)# loop over the face detections循环面部检测for rect in rects:# compute the bounding box of the face and draw it on the# frame# 计算面部的边界框并将其绘制在帧上(bX, bY, bW, bH) = face_utils.rect_to_bb(rect)#cv2.rectangle(frame, (bX, bY), (bX + bW, bY + bH), (0, 255, 0), 1)# determine the facial landmarks for the face region, then# convert the facial landmark (x, y)-coordinates to a NumPy# array# 确定面部区域的面部特征点，然后将面部坐标(x, y)坐标转换为NumPy数组shape = predictor(gray, rect)shape = face_utils.shape_to_np(shape)# extract the left and right eye coordinates, then use the# coordinates to compute the eye aspect ratio for both eyes# 提取左右眼坐标，然后使用坐标计算双眼的长宽比leftEye = shape[lStart:lEnd]rightEye = shape[rStart:rEnd]leftEAR = eye_aspect_ratio(leftEye)rightEAR = eye_aspect_ratio(rightEye)# average the eye aspect ratio together for both eyes# 将两只眼睛的长宽比平均ear = (leftEAR + rightEAR) / 2.0# check to see if the eye aspect ratio is below the blink# threshold, and if so, increment the blink frame counter# 检查眼睛宽高比是否低于眨眼阈值，如果是，增加眨眼帧计数器if ear < EYE_AR_THRESH:COUNTER += 1# if the eyes were closed for a sufficient number of times# then show the warning# 如果眼睛闭上足够多的次数，然后显示警告if COUNTER >= EYE_AR_CONSEC_FRAMES:cv2.putText(frame, "Eyes Closed!", (500, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)# otherwise, the eye aspect ratio is not below the blink# threshold, so reset the counter and alarm# 否则，眼睛宽高比不低于眨眼阈值，因此重置计数器和报警else:COUNTER = 0mouth = shape[mStart:mEnd]mouthMAR = mouth_aspect_ratio(mouth)mar = mouthMAR# Draw text if mouth is openif mar > MOUTH_AR_THRESH:cv2.putText(frame, "Yawning!", (800, 20),cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)# show the frame qcv2.imshow("Frame", frame)key = cv2.waitKey(1) & 0xFF# if the `q` key was pressed, break from the loopif key == ord("q"):break# print(image_points)# do a bit of cleanup
cv2.destroyAllWindows()
vs.stop()