目录
一、任务目标
- 将虚拟主播实验中眼动的数据(眼睛移动、头部姿态等数据),生成到Flask网页前端
- 实现一个头部姿态左右转动和上下抬头低头的检测,驱动一个vue3直方图(或其他图形),实现头部运动交互的数据可视化作品
- 实现一个头部姿态左右转动和上下抬头低头的检测,驱动数据可视化切换数据(Vue3网页,或者普通网页均可)
二、程序实现
1. Python代码
创建Flask服务,构建Flask页面。
1) 导入库
from flask import Flask, render_template, jsonify
from flask_sqlalchemy import SQLAlchemy
import pymysql
from flask import Flask, json, request, Response, render_template
from flask_cors import CORS
import json
import configparser
import os
import cv2
import numpy as np
import dlib
2) 设置
设置连接数据库链接,便于后续存取可视化数据。注意,一定要加入“app.config[‘JSON_AS_ASCII’] = False”,从而确保页面显示为utf-8格式。
app = Flask(__name__)
app.config['SECRET_KEY'] = 'hard to guess string'
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://root:123456@localhost:3306/7'
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False
app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True
app.config['JSON_AS_ASCII'] = False
db = SQLAlchemy(app)
3) 脸部数据存储的类
在Flask中定义数据库的列,分别是:图片中最大人脸的范围、水平旋转量、垂直旋转量。
class face_data(db.Model):
id = db.Column(db.Integer, primary_key=True)
face = db.Column(db.String(255))
new1 = db.Column(db.String(255))
new2 = db.Column(db.String(255))
4) 主页
构建Flask页面中的主页。
@app.route("/")
def hello():
return render_template("home.html")
5) save页
构建Flask页面中的数据存储save页。
@app.route("/save", methods=["GET", "POST"])
def save_data():
"""
detect face
"""
import cv2
import numpy as np
# to detect face key point
import dlib
DETECTOR = dlib.get_frontal_face_detector()
# 人脸模型数据
PREDICTOR = dlib.shape_predictor(
'./static/shape_predictor_68_face_landmarks.dat')
def face_positioning(img):
"""
定位人脸
计算最大面积
"""
dets = DETECTOR(img, 0)
if not dets:
return None
return max(dets, key=lambda det: (det.right() - det.left()) * (det.bottom() - det.top()))
def extract_key_points(img, position):
"""
提取关键点
"""
landmark_shape = PREDICTOR(img, position)
key_points = []
for i in range(68):
pos = landmark_shape.part(i)
key_points.append(np.array([pos.x, pos.y], dtype=np.float32))
return key_points
def generate_points(key_points):
"""
生成构造点
"""
def center(array):
return sum([key_points[i] for i in array]) / len(array)
left_brow = [18, 19, 20, 21]
right_brow = [22, 23, 24, 25]
# 下巴
chin = [6, 7, 8, 9, 10]
nose = [29, 30]
return center(left_brow + right_brow), center(chin), center(nose)
def generate_features(contruction_points):
"""
生成特征
"""
brow_center, chin_center, nose_center = contruction_points
mid_edge = brow_center - chin_center
# 斜边
bevel_edge = brow_center - nose_center
mid_edge_length = np.linalg.norm(mid_edge)
# 高与底的比值
horizontal_rotation = np.cross(
mid_edge, bevel_edge) / mid_edge_length ** 2
# @ 点乘
vertical_rotation = mid_edge @ bevel_edge / mid_edge_length ** 2
return np.array([horizontal_rotation, vertical_rotation])
def draw_image(h_rotation, v_rotation):
"""
画脸
Args:
h_rotation: 水平旋转量
v_rotation: 垂直旋转量
"""
img = np.ones([512, 512], dtype=np.float32)
face_length = 200
center = 256, 256
left_eye = int(220 - h_rotation *
face_length), int(249 + v_rotation * face_length)
right_eye = int(292 - h_rotation *
face_length), int(249 + v_rotation * face_length)
month = int(256 - h_rotation * face_length /
2), int(310 + v_rotation * face_length / 2)
cv2.circle(img, center, 100, 0, 1)
cv2.circle(img, left_eye, 15, 0, 1)
cv2.circle(img, right_eye, 15, 0, 1)
cv2.circle(img, month, 5, 0, 1)
return img
def extract_img_features(img):
"""
提取图片特征
"""
face_position = face_positioning(img)
if not face_position:
cv2.imshow('self', img)
cv2.waitKey(1)
return None
key_points = extract_key_points(img, face_position)
for i, (p_x, p_y) in enumerate(key_points):
cv2.putText(img, str(i), (int(p_x), int(p_y)),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (255, 255, 255))
construction_points = generate_points(key_points)
for i, (p_x, p_y) in enumerate(construction_points):
cv2.putText(img, str(i), (int(p_x), int(p_y)),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (255, 255, 255))
rotation = generate_features(construction_points)
cv2.putText(img, str(rotation),
(int(construction_points[-1][0]),
int(construction_points[-1][1])),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 255))
cv2.imshow('self', img)
return face_position, rotation
if __name__ == '__main__':
CAP = cv2.VideoCapture(0+ cv2.CAP_DSHOW) #
# 原点特征组 my front side
ORIGIN_FEATURE_GROUP = [-0.00899233, 0.39529446]
FEATURE_GROUP = [0, 0]
while True:
RETVAL, IMAGE = CAP.read()
# 翻转视频
IMAGE = cv2.flip(IMAGE, 1)
face_position,NEW_FEATURE_GROUP = extract_img_features(IMAGE) #
if NEW_FEATURE_GROUP is not None:
FEATURE_GROUP = NEW_FEATURE_GROUP - ORIGIN_FEATURE_GROUP
HORI_ROTATION, VERT_ROTATION = FEATURE_GROUP
res = face_data() # 实例化一条记录
res.face = face_position
res.new1 = NEW_FEATURE_GROUP[0]
res.new2 = NEW_FEATURE_GROUP[1]
db.session.add(res) # 逻辑添加
db.session.commit() # 添加记录
if cv2.waitKey(1) & 0xFF == ord('q'):
print("close")
break
return "数据存储成功!"
6) show页
构建Flask页面中数据显示show页
@app.route('/show')
def show_data():
db = pymysql.connect(host='localhost', user='root', password='123456', database='7', charset='utf8')
cursor = db.cursor()
cursor.execute("Select new1,new2 from face_data")
rs = cursor.fetchall()
rs = list(rs)
print(rs[0:10])
return render_template("show_data.html", rs=rs)
7) 数据库连接类
class Mysql(object):
def __init__(self):
try:
self.conn = pymysql.connect(host='localhost', user='root', password='123456', database='7',charset='utf8')
self.cursor = self.conn.cursor() # 用来获得python执行Mysql命令的方法(游标操作)
print("成功")
except:
print("失败")
def getItems(self):
sql = "select id,new1,new2 from face_data" # 获取food数据表的内容
self.cursor.execute(sql)
items = self.cursor.fetchall() # 接收全部的返回结果行
return items
8) line页
构建Flask页面中的line页,用于显示数据的可视化结果。
@app.route('/line')
def line():
db = Mysql()
items = db.getItems()
return render_template('virtual_echart.html', items=items)
9) face页
构建Flask页面中的face页,用于显示原始人脸视频结果。
def get_frames1():
CAP = cv2.VideoCapture(0)
ORIGIN_FEATURE_GROUP = [-0.00899233, 0.39529446]
FEATURE_GROUP = [0, 0]
while True:
RETVAL, IMAGE = CAP.read()
if not RETVAL:
break
IMAGE = cv2.flip(IMAGE, 1)
NEW_FEATURE_GROUP = extract_img_features(IMAGE)
if NEW_FEATURE_GROUP is not None:
FEATURE_GROUP = NEW_FEATURE_GROUP - ORIGIN_FEATURE_GROUP
HORI_ROTATION, VERT_ROTATION = FEATURE_GROUP
frame=cv2.imencode('.jpg',IMAGE)[1].tobytes()
yield (b'--frame\r\n'
b'Content-Type: image/jpeg\r\n\r\n' + frame + b'\r\n')
@app.route('/face')
def face():
return render_template("index.html")
@app.route('/face1')
def face1():
return Response(get_frames1(),mimetype='multipart/x-mixed-replace; boundary=frame')
10) 加载人脸模型数据
DETECTOR = dlib.get_frontal_face_detector()
PREDICTOR = dlib.shape_predictor(
'./static/shape_predictor_68_face_landmarks.dat')
11) line页
构建Flask页面中的line页,用于显示数据的可视化结果。
@app.route('/line')
def line():
db = Mysql()
items = db.getItems()
return render_template('virtual_echart.html', items=items)
12) 提取人脸特征子函数
def face_positioning(img):
dets = DETECTOR(img, 0)
if not dets:
return None
return max(dets, key=lambda det: (det.right() - det.left()) * (det.bottom() - det.top()))
def extract_key_points(img, position):
landmark_shape = PREDICTOR(img, position)
key_points = []
for i in range(68):
pos = landmark_shape.part(i)
key_points.append(np.array([pos.x, pos.y], dtype=np.float32))
return key_points
def generate_points(key_points):
def center(array):
return sum([key_points[i] for i in array]) / len(array)
left_brow = [18, 19, 20, 21]
right_brow = [22, 23, 24, 25]
# 下巴
chin = [6, 7, 8, 9, 10]
nose = [29, 30]
return center(left_brow + right_brow), center(chin), center(nose)
def generate_features(contruction_points):
brow_center, chin_center, nose_center = contruction_points
mid_edge = brow_center - chin_center
bevel_edge = brow_center - nose_center
mid_edge_length = np.linalg.norm(mid_edge)
horizontal_rotation = np.cross(
mid_edge, bevel_edge) / mid_edge_length ** 2
vertical_rotation = mid_edge @ bevel_edge / mid_edge_length ** 2
return np.array([horizontal_rotation, vertical_rotation])
def draw_image(h_rotation, v_rotation):
img = np.ones([512, 512], dtype=np.float32)
face_length = 200
center = 256, 256
left_eye = int(220 - h_rotation *
face_length), int(249 + v_rotation * face_length)
right_eye = int(292 - h_rotation *
face_length), int(249 + v_rotation * face_length)
month = int(256 - h_rotation * face_length /
2), int(310 + v_rotation * face_length / 2)
cv2.circle(img, center, 100, 0, 1)
cv2.circle(img, left_eye, 15, 0, 1)
cv2.circle(img, right_eye, 15, 0, 1)
cv2.circle(img, month, 5, 0, 1)
return img
def extract_img_features(img):
face_position = face_positioning(img)
print("图片中最大人脸的范围为:", face_position)
if not face_position:
cv2.imshow('self', img)
cv2.waitKey(1)
return None
key_points = extract_key_points(img, face_position)
print("68个关键点的位置为:", key_points)
for i, (p_x, p_y) in enumerate(key_points):
cv2.putText(img, str(i), (int(p_x), int(p_y)),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (255, 255, 255))
construction_points = generate_points(key_points)
print("生成的构造点的位置为:", construction_points)
for i, (p_x, p_y) in enumerate(construction_points):
cv2.putText(img, str(i), (int(p_x), int(p_y)),
cv2.FONT_HERSHEY_COMPLEX, 0.25, (255, 255, 255))
rotation = generate_features(construction_points)
print("水平旋转量和垂直旋转量为:", rotation)
cv2.putText(img, str(rotation),
(int(construction_points[-1][0]),
int(construction_points[-1][1])),
cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 255))
cv2.imshow('self', img)
return rotation
13) 主函数
if __name__ == '__main__':
app.run(debug=True)
2. html代码
1) home.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>人脸关键数据数据提取</title>
</head>
<body>
<h1 align = "center" style = "font-family: 宋体"> 人脸关键数据数据提取 </h1>
<p><a href = "{{ url_for('save_data') }}" rel = "external nofollow"
style = "top: 200px; text-decoration:none; font-size: 15px; color:black;font-family: 宋体"> 打开摄像头提取人脸数据 </a></p>
<p><a href = "{{ url_for('face1') }}" rel = "external nofollow"
style = "top: 200px; text-decoration:none; font-size: 15px; color:black;font-family: 宋体"> 将视频传入网页 </a></p>
<p><a href = "{{ url_for('show_data') }}" rel = "external nofollow"
style = "top: 200px;left:80px; text-decoration: none; font-size: 15px; color:black;font-family: 宋体"> 人脸关键数据 </a></p>
<p><a href = "{{ url_for('line') }}" rel = "external nofollow"
style = "top: 200px; left:200px;text-decoration: none; font-size: 15px; color:black; font-family: 宋体"> 折线图 </a></p>
</body>
</html>
2) index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>camera</title>
<style>
.con{
float:left;
height:600px;width:600px;
margin-right:10px;
}
</style>
</head>
<body>
<h3>接受传输来的视频</h3>
<div class="con">
< img src="{{url_for('face1')}}" alt="" width="100%">
</div>
</body>
</html>
3) show_data.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>人脸关键数据</title>
</head>
<body>
<h1>水平旋转量 垂直旋转量</h1>
{% for r in rs %}
{{r}}<br>
{% endfor %}
</body>
</html>
4) virtual_echart.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>折线图</title>
<script src="/static/js/echarts.min.js"></script>
<style>
#main{
width: 1200px;
height: 400px;
position:relative;
left:0px;
top:80px;
}
</style>
</head>
<body>
<h1 align="center" style="font-family:宋体;font-size: 36px">折线图</h1>
<div id="main" ></div>
<script type="text/javascript">
// 基于准备好的dom,初始化 echarts 实例并绘制图表。
var myChart=echarts.init(document.getElementById('main'));
// 指定图表的配置项和数据
var option = {
title: {
text: ''
},
legend:{
data:['horizontal','vertical']
},
dataZoom: [{
type: 'slider',
show: true, //flase直接隐藏图形
xAxisIndex: [0],
left: '9%', //滚动条靠左侧的百分比
bottom: -5,
start: 1,//滚动条的起始位置
end: 40//滚动条的截止位置(按比例分割你的柱状图x轴长度)
}],
xAxis: {
name:"data-id",
type: 'category',
data:[
{% for item in items %}
"{{ item[0]}}",
{% endfor %}
]},
yAxis: {
name:"data-rotate",
type: 'value',
axisLabel : {
formatter: '{value} '}
},
series: [
{
name:'horizontal',
type: 'line', //line折线图。bar柱形图
data:[{% for item in items %}
"{{ item[1]}}",
{% endfor %}],
itemStyle:{normal: {color:" #ADFF2F"}}
},
{
name:'vertical',
type: 'line', //line折线图。bar柱形图
data:[{% for item in items %}
"{{ item[2]}}",
{% endfor %}],
itemStyle:{normal: {color:" #FFFF00"}}
},
]
};
myChart.setOption(option);// 基于准备好的dom,初始化 echarts 实例并绘制图表。
</script>
</body>
</html>
总结
将头部特征视频显示到网页上,又将头部姿态数据实时存储到数据库里,在网页打印出相应数据, 使用echarts绘制折线图。
在查阅资料并解决遇到的问题后,目前任务已全部完成,由于结果视频演示中有真人的脸就不在博客中展示了,视频演示已发在课程群里中。