文章目录
『opencv』笔记1:基于 OpenCV 的图像融合
1 OpenCV显著性算法背景介绍
1.1 OpenCV显著性检测算法相关信息介绍
OpenCV contrib 库中的 saliency 模块提供四种显著性检测算法。本节主要介绍这四种方法的相关信息。
1.1.1 Static Saliency Spectral Residual
- 原理:该方法从自然图像统计原理出发,模拟注意前视觉搜索的行为。该算法对每幅图像的对数谱进行分析,得到谱残差SR (Spectral Residual)。然后将谱残差进行空间变换,得到显著性图,该显著性图显示了感兴趣目标的位置。
- 论文:Saliency Detection: A Spectral Residual Approach
- 检测方式:单张图片检测
实现方式
// c++ demo
saliencyAlgorithm = StaticSaliencySpectralResidual::create();
bool success = saliencyAlgorithm -> computeSaliency(image, saliencyMap); // 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.StaticSaliencySpectralResidual_create()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image) # 计算显著性
1.1.2 Static Saliency Fine Grained
- 原理:该方法基于空间尺度差异 center-surround differences 计算显著性。利用积分图像 integral images.实时生成高分辨率显著性图。
- 论文:Human detection using a mobile platform and novel features derived from a visual saliency mechanism
- 检测方式:单张图片检测
实现方式
// c++ demo
saliencyAlgorithm = StaticSaliencyFineGrained::create();
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap); // 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.StaticSaliencyFineGrained_create()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image) # 计算显著性
1.1.3 Objectness BING
- 原理:基于二值化梯度特征(BING features)进行物体检测
- 论文:BING: Binarized Normed Gradients for Objectness Estimation at 300fps
- 检测方式:加载模型,单张图片检测
实现方式
// c++ demo
saliencyAlgorithm = ObjectnessBING::create();
vector<Vec4i> saliencyMap;
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setTrainingPath(training_path); // 提取模型文件参数
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setBBResDir("Results"); // 将算法检测结果保存在Results文件夹内
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap); // 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.ObjectnessBING_create()
saliencyAlgorithm.setTrainingPath(training_path) # 提取模型文件参数
saliencyAlgorithm.setBBResDir("Results") # 将算法检测结果保存在Results文件夹内
success, saliencyMap = saliencyAlgorithm.computeSaliency(image) # 计算显著性
1.1.4 BinWangApr2014
- 原理:基于运动背景减除法实现显著性区域检测
- 论文:A Fast Self-Tuning Background Subtraction Algorithm
- 检测方式:基于多张图片初始化模型,然后进行单张图片检测
实现方式
// c++ demo
saliencyAlgorithm = MotionSaliencyBinWangApr2014::create();
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->setImagesize(image.cols, image.rows); // 设置数据结构大小
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->init(); // 初始化
saliencyAlgorithm->computeSaliency(frame, saliencyMap);
# python demo
saliencyAlgorithm = cv2.saliency.MotionSaliencyBinWangApr2014_create()
saliencyAlgorithm.setImagesize(image.shape[1], image.shape[0]) # 设置数据结构大小
saliencyAlgorithm.init() # 初始化
success, saliencyMap = saliencyAlgorithm.computeSaliency(frame)
1.2 OpenCV saliency 模块整体说明
显著性检测算法与目标检测算法大大不同。显著性检测算法,只是判断图中有显著目标的区域,这些区域可能包含目标也可能不包含目标,因方法而异。类比人眼的观察方式,显著性检测算法是许多计算机视觉任务的第一步,检测出显著性区域后,对这些显著性区域进行进一步判断和预测。显著性检测算法通常检测速度较快,某些计算量大的算法如深度学习图像分类算法,可以只在显著性区域上运行,以缩小检测范围,加快检测速度,提高检测精度。
OpenCV saliency模块提供了四种不同的显著性检测方法,但是按方法类别只有三种。OpenCV saliency模块的类关系如下图所示:
OpenCV saliency模块提供的三种不同方法类别模块介绍如下:
- Motion saliency模块:这类算法输入为连续的图像帧,通过运动检测算法对连续图像帧进行处理,然后对运动目标进行跟踪,最终将运动目标设置为显著区域。代表为
BinWangApr2014
算法。该类算法容易出现丢帧和鬼影情况,运动检测效果不如主流的运动检测算法,实际图像显著性检测效果一般。 - Objectness模块:这类算法输入为单帧图像,通过计算得到大量的建议区域,并将这些建议区域作为显著性区域。代表为
ObjectnessBING
算法。该类算法检测速度较慢,实际检测出来的建议区域可能上万个,需要进行筛选,总体效果一般。 - Static saliency模块:这类算法输入为单帧图像,通过图像特征和统计量来定位图像中的显著性区域。代表为
Static Saliency Spectral Residual
和Static Saliency Fine Grained
。该类算法检测速度非常快,不过效果总体一般。
2 代码实现
本文所提供的代码可以对视频或者图像进行显著性检测,BinWangApr2014 只能对视频进行显著性检测。本文提供 C++ 和 Python 代码实现,代码如下:
C++
#include <opencv2/opencv.hpp>
#include <opencv2/saliency.hpp>
#include <iostream>
using namespace std;
using namespace cv;
using namespace saliency;
int main()
{
// 显著性检测算法
// 可选:SPECTRAL_RESIDUAL,FINE_GRAINED,BING,BinWangApr2014
String saliency_algorithm = "FINE_GRAINED";
String video_name = "video/vtest.avi"; // 检测视频或者图像
// String video_name = "video/dog.jpg";
int start_frame = 0; // 起始帧
String training_path = "ObjectnessTrainedModel"; // 模型路径
// 如果算法名和视频名为空,停止检测
if (saliency_algorithm.empty() || video_name.empty())
{
cout << "Please set saliency_algorithm and video_name";
return -1;
}
VideoCapture cap;
cap.open(video_name);
cap.set(CAP_PROP_POS_FRAMES, start_frame); // 设置视频起始帧
Mat frame; // 输入图像
// 实例化 saliencyAlgorithm 结构
Ptr<Saliency> saliencyAlgorithm;
Mat binaryMap; // 二值化检测结果
Mat image; // 检测图像
cap >> frame; // 读图
if (frame.empty())
{
return 0;
}
frame.copyTo(image);
// 根据输入的方法确定检测类型
// StaticSaliencySpectralResidual
if (saliency_algorithm.find("SPECTRAL_RESIDUAL") == 0)
{
Mat saliencyMap; // 检测结果,白色区域表示显著区域
saliencyAlgorithm = StaticSaliencySpectralResidual::create();
double start = static_cast<double>(getTickCount()); // 计算显著性
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;
if (success)
{
StaticSaliencySpectralResidual spec;
double start = static_cast<double>(getTickCount()); // 二值化图像
spec.computeBinaryMap(saliencyMap, binaryMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeBinaryMap cost time is: " << duration * 1000 << "ms" << endl;
imshow("Original Image", image);
imshow("Saliency Map", saliencyMap);
imshow("Binary Map", binaryMap);
saliencyMap.convertTo(saliencyMap, CV_8UC3, 256); // 转换格式才能保存图片
imwrite("Results/SPECTRAL_RESIDUAL_saliencyMap.jpg", saliencyMap);
imwrite("Results/SPECTRAL_RESIDUAL_binaryMap.jpg", binaryMap);
waitKey(0);
}
}
// StaticSaliencyFineGrained
else if (saliency_algorithm.find("FINE_GRAINED") == 0)
{
Mat saliencyMap;
saliencyAlgorithm = StaticSaliencyFineGrained::create();
double start = static_cast<double>(getTickCount()); // 计算显著性
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;
if (success)
{
StaticSaliencyFineGrained spec;
double start = static_cast<double>(getTickCount()); // 二值化图像
spec.computeBinaryMap(saliencyMap, binaryMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeBinaryMap cost time is: " << duration * 1000 << "ms" << endl;
imshow("Saliency Map", saliencyMap);
imshow("Original Image", image);
imshow("Binary Map", binaryMap);
saliencyMap.convertTo(saliencyMap, CV_8UC3, 256); // 转换格式才能保存图片
imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap);
imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap);
waitKey(0);
}
}
// ObjectnessBING
else if (saliency_algorithm.find("BING") == 0)
{
// 判断模型是否存在
if (training_path.empty())
{
cout << "Path of trained files missing! " << endl;
return -1;
}
else
{
saliencyAlgorithm = ObjectnessBING::create();
vector<Vec4i> saliencyMap;
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setTrainingPath(training_path); // 提取模型文件参数
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setBBResDir("Results"); // 将算法检测结果保存在Results文件夹内
double start = static_cast<double>(getTickCount()); // 计算显著性
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;
if (success)
{
int ndet = int(saliencyMap.size()); // saliencyMap获取检测到的目标个数
std::cout << "Objectness done " << ndet << std::endl;
// 目标按可能性从大到小排列,maxd为显示前5个目标,step设置颜色,jitter设置矩形框微调
int maxd = 5, step = 255 / maxd, jitter = 9;
Mat draw = image.clone();
for (int i = 0; i < std::min(maxd, ndet); i++)
{
Vec4i bb = saliencyMap[i]; // 获得矩形框坐标点
Scalar col = Scalar(((i*step) % 255), 50, 255 - ((i*step) % 255)); // 设定颜色
Point off(theRNG().uniform(-jitter, jitter), theRNG().uniform(-jitter, jitter)); // 矩形框微调
rectangle(draw, Point(bb[0] + off.x, bb[1] + off.y), Point(bb[2] + off.x, bb[3] + off.y), col, 2); // 画矩形
rectangle(draw, Rect(20, 20 + i * 10, 10, 10), col, -1); // 颜色标注
}
imshow("BING", draw);
imwrite("Results/BING_draw.jpg", draw);
waitKey();
}
else
{
std::cout << "No saliency found for " << video_name << std::endl;
}
}
}
// BinWangApr2014
else if (saliency_algorithm.find("BinWangApr2014") == 0)
{
saliencyAlgorithm = MotionSaliencyBinWangApr2014::create();
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->setImagesize(image.cols, image.rows); // 设置数据结构大小
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->init(); // 初始化
bool paused = false;
for (;; )
{
if (!paused)
{
cap >> frame;
if (frame.empty())
{
return 0;
}
cvtColor(frame, frame, COLOR_BGR2GRAY);
Mat saliencyMap;
double start = static_cast<double>(getTickCount());
saliencyAlgorithm->computeSaliency(frame, saliencyMap);
double duration = ((double)getTickCount() - start) / getTickFrequency();
cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;
imshow("image", frame);
imshow("saliencyMap", saliencyMap * 255);
}
char c = (char)waitKey(2);
if (c == 'q')
break;
if (c == 'p')
paused = !paused;
}
}
destroyAllWindows();
return 0;
}
python
# -*- coding: utf-8 -*-
import cv2
import random
def main():
# 显著性检测算法
# 可选:SPECTRAL_RESIDUAL,FINE_GRAINED,BING,BinWangApr2014
saliency_algorithm = "FINE_GRAINED"
video_name = "video/vtest.avi"
# video_name = "video/dog.jpg";
start_frame = 0 # 起始帧
training_path = "ObjectnessTrainedModel" # 模型路径
# 如果算法名和视频名为空,停止检测
if saliency_algorithm is None or video_name is None:
print("Please set saliency_algorithm and video_name")
return
cap = cv2.VideoCapture(video_name)
cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame) # 设置视频起始帧
_, frame = cap.read()
if frame is None:
print("Please set saliency_algorithm and video_name")
return
image = frame.copy()
# 根据输入的方法确定检测类型
if saliency_algorithm.find("SPECTRAL_RESIDUAL") == 0:
# 检测结果,白色区域表示显著区域
saliencyAlgorithm = cv2.saliency.StaticSaliencySpectralResidual_create()
start = cv2.getTickCount() # 计算显著性
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
if success:
start = cv2.getTickCount() # 二值化图像
_, binaryMap = saliencyAlgorithm.computeBinaryMap(saliencyMap)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
cv2.imshow("Saliency Map", saliencyMap)
cv2.imshow("Original Image", image)
cv2.imshow("Binary Map", binaryMap)
saliencyMap = (saliencyMap * 255) # 转换格式才能保存图片
cv2.imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap)
cv2.imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap)
cv2.waitKey(0)
# FINE_GRAINED
elif saliency_algorithm.find("FINE_GRAINED") == 0:
saliencyAlgorithm = cv2.saliency.StaticSaliencyFineGrained_create()
start = cv2.getTickCount()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
if success:
start = cv2.getTickCount()
_, binaryMap = saliencyAlgorithm.computeBinaryMap(saliencyMap)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
cv2.imshow("Saliency Map", saliencyMap)
cv2.imshow("Original Image", image)
cv2.imshow("Binary Map", binaryMap)
saliencyMap = (saliencyMap * 255)
cv2.imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap)
cv2.imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap)
cv2.waitKey(0)
elif saliency_algorithm.find("BING") == 0:
# 判断模型是否存在
if training_path is None:
print("Path of trained files missing! ")
return
else:
saliencyAlgorithm = cv2.saliency.ObjectnessBING_create()
saliencyAlgorithm.setTrainingPath(training_path) # 提取模型文件参数
saliencyAlgorithm.setBBResDir("Results") # 将算法检测结果保存在Results文件夹内
start = cv2.getTickCount()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
if success:
ndet = saliencyMap.shape[0] # saliencyMap 获取检测到的目标个数
print("Objectness done ", ndet)
# 目标按可能性从大到小排列,maxd为显示前5个目标,step设置颜色,jitter设置矩形框微调
maxd = 5
step = 255 / maxd
jitter = 9
draw = image.copy()
for i in range(0, min(maxd, ndet)):
bb = saliencyMap[i][0]
col = ((i * step) % 255), 50, 255 - ((i * step) % 255)
off = random.randint(-jitter, jitter), random.randint(-jitter, jitter)
cv2.rectangle(draw, (bb[0] + off[0], bb[1] + off[1]), (bb[2] + off[0], bb[3] + off[1]), col, 2)
cv2.rectangle(draw, (20, 20 + i * 10, 10, 10), col, -1)
cv2.imwrite("Results/BING_draw.jpg", draw)
cv2.imshow("BING", draw)
cv2.waitKey(0)
# 需要传入图像建模
elif saliency_algorithm.find("BinWangApr2014") == 0:
saliencyAlgorithm = cv2.saliency.MotionSaliencyBinWangApr2014_create()
saliencyAlgorithm.setImagesize(image.shape[1], image.shape[0]) # 设置数据结构大小
saliencyAlgorithm.init() # 初始化
paused = False
while True:
if not paused:
_, frame = cap.read()
if frame is None:
break
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
start = cv2.getTickCount()
success, saliencyMap = saliencyAlgorithm.computeSaliency(frame)
duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
cv2.imshow("image", frame)
cv2.imshow("saliencyMap", saliencyMap * 255)
c = cv2.waitKey(2)
c = chr(c) if c != -1 else 0
if c == 'q':
break
if c == 'p':
paused = not paused
cv2.destroyAllWindows()
return
if __name__ == '__main__':
main()