『opencv』笔记2:基于OpenCV的显著性绘制


『opencv』笔记1:基于 OpenCV 的图像融合


1 OpenCV显著性算法背景介绍

1.1 OpenCV显著性检测算法相关信息介绍

OpenCV contrib 库中的 saliency 模块提供四种显著性检测算法。本节主要介绍这四种方法的相关信息。

1.1.1 Static Saliency Spectral Residual

  • 原理:该方法从自然图像统计原理出发,模拟注意前视觉搜索的行为。该算法对每幅图像的对数谱进行分析,得到谱残差SR (Spectral Residual)。然后将谱残差进行空间变换,得到显著性图,该显著性图显示了感兴趣目标的位置。
  • 论文:Saliency Detection: A Spectral Residual Approach
  • 检测方式:单张图片检测

实现方式

// c++ demo
saliencyAlgorithm = StaticSaliencySpectralResidual::create();
bool success = saliencyAlgorithm -> computeSaliency(image, saliencyMap);		// 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.StaticSaliencySpectralResidual_create()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)		# 计算显著性

1.1.2 Static Saliency Fine Grained

实现方式

// c++ demo
saliencyAlgorithm = StaticSaliencyFineGrained::create();
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);		// 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.StaticSaliencyFineGrained_create()
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)		# 计算显著性

1.1.3 Objectness BING

实现方式

// c++ demo
saliencyAlgorithm = ObjectnessBING::create();
vector<Vec4i> saliencyMap;
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setTrainingPath(training_path);	// 提取模型文件参数
saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setBBResDir("Results");	// 将算法检测结果保存在Results文件夹内
bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);	// 计算显著性
# python demo
saliencyAlgorithm = cv2.saliency.ObjectnessBING_create()
saliencyAlgorithm.setTrainingPath(training_path)	# 提取模型文件参数
saliencyAlgorithm.setBBResDir("Results")	# 将算法检测结果保存在Results文件夹内
success, saliencyMap = saliencyAlgorithm.computeSaliency(image)	# 计算显著性

1.1.4 BinWangApr2014

实现方式

// c++ demo
saliencyAlgorithm = MotionSaliencyBinWangApr2014::create();
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->setImagesize(image.cols, image.rows);	// 设置数据结构大小
saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->init();	// 初始化
saliencyAlgorithm->computeSaliency(frame, saliencyMap);
# python demo
saliencyAlgorithm = cv2.saliency.MotionSaliencyBinWangApr2014_create()
saliencyAlgorithm.setImagesize(image.shape[1], image.shape[0])	# 设置数据结构大小
saliencyAlgorithm.init()	# 初始化
success, saliencyMap = saliencyAlgorithm.computeSaliency(frame)

1.2 OpenCV saliency 模块整体说明

显著性检测算法与目标检测算法大大不同。显著性检测算法,只是判断图中有显著目标的区域,这些区域可能包含目标也可能不包含目标,因方法而异。类比人眼的观察方式,显著性检测算法是许多计算机视觉任务的第一步,检测出显著性区域后,对这些显著性区域进行进一步判断和预测。显著性检测算法通常检测速度较快,某些计算量大的算法如深度学习图像分类算法,可以只在显著性区域上运行,以缩小检测范围,加快检测速度,提高检测精度。

OpenCV saliency模块提供了四种不同的显著性检测方法,但是按方法类别只有三种。OpenCV saliency模块的类关系如下图所示:
在这里插入图片描述
OpenCV saliency模块提供的三种不同方法类别模块介绍如下:

  • Motion saliency模块:这类算法输入为连续的图像帧,通过运动检测算法对连续图像帧进行处理,然后对运动目标进行跟踪,最终将运动目标设置为显著区域。代表为 BinWangApr2014 算法。该类算法容易出现丢帧和鬼影情况,运动检测效果不如主流的运动检测算法,实际图像显著性检测效果一般。
  • Objectness模块:这类算法输入为单帧图像,通过计算得到大量的建议区域,并将这些建议区域作为显著性区域。代表为 ObjectnessBING 算法。该类算法检测速度较慢,实际检测出来的建议区域可能上万个,需要进行筛选,总体效果一般。
  • Static saliency模块:这类算法输入为单帧图像,通过图像特征和统计量来定位图像中的显著性区域。代表为Static Saliency Spectral ResidualStatic Saliency Fine Grained。该类算法检测速度非常快,不过效果总体一般。

2 代码实现

本文所提供的代码可以对视频或者图像进行显著性检测,BinWangApr2014 只能对视频进行显著性检测。本文提供 C++ 和 Python 代码实现,代码如下:

C++

#include <opencv2/opencv.hpp>
#include <opencv2/saliency.hpp>
#include <iostream>

using namespace std;
using namespace cv;
using namespace saliency;

int main()
{
	// 显著性检测算法
	// 可选:SPECTRAL_RESIDUAL,FINE_GRAINED,BING,BinWangApr2014
	String saliency_algorithm = "FINE_GRAINED";
	String video_name = "video/vtest.avi";	// 检测视频或者图像
	// String video_name = "video/dog.jpg";
	int start_frame = 0;	// 起始帧
	String training_path = "ObjectnessTrainedModel";	// 模型路径
	// 如果算法名和视频名为空,停止检测
	if (saliency_algorithm.empty() || video_name.empty())
	{
		cout << "Please set saliency_algorithm and video_name";
		return -1;
	}

	VideoCapture cap;
	cap.open(video_name);
	cap.set(CAP_PROP_POS_FRAMES, start_frame);	// 设置视频起始帧

	Mat frame;	// 输入图像

	// 实例化 saliencyAlgorithm 结构
	Ptr<Saliency> saliencyAlgorithm;

	Mat binaryMap;	// 二值化检测结果
	Mat image;	// 检测图像

	cap >> frame;	// 读图
	if (frame.empty())
	{
		return 0;
	}

	frame.copyTo(image);

	// 根据输入的方法确定检测类型
	// StaticSaliencySpectralResidual
	if (saliency_algorithm.find("SPECTRAL_RESIDUAL") == 0)
	{
		Mat saliencyMap;	// 检测结果,白色区域表示显著区域
		saliencyAlgorithm = StaticSaliencySpectralResidual::create();
		double start = static_cast<double>(getTickCount());	// 计算显著性
		bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
		double duration = ((double)getTickCount() - start) / getTickFrequency();
		cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;

		if (success)
		{
			StaticSaliencySpectralResidual spec;
			double start = static_cast<double>(getTickCount());	// 二值化图像
			spec.computeBinaryMap(saliencyMap, binaryMap);
			double duration = ((double)getTickCount() - start) / getTickFrequency();
			cout << "computeBinaryMap cost time is: " << duration * 1000 << "ms" << endl;

			imshow("Original Image", image);
			imshow("Saliency Map", saliencyMap);
			imshow("Binary Map", binaryMap);

			saliencyMap.convertTo(saliencyMap, CV_8UC3, 256);	// 转换格式才能保存图片
			imwrite("Results/SPECTRAL_RESIDUAL_saliencyMap.jpg", saliencyMap);
			imwrite("Results/SPECTRAL_RESIDUAL_binaryMap.jpg", binaryMap);
			waitKey(0);
		}
	}

	// StaticSaliencyFineGrained
	else if (saliency_algorithm.find("FINE_GRAINED") == 0)
	{
		Mat saliencyMap;
		saliencyAlgorithm = StaticSaliencyFineGrained::create();
		double start = static_cast<double>(getTickCount());	// 计算显著性
		bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
		double duration = ((double)getTickCount() - start) / getTickFrequency();
		cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;

		if (success)
		{
			StaticSaliencyFineGrained spec;
			double start = static_cast<double>(getTickCount());	// 二值化图像
			spec.computeBinaryMap(saliencyMap, binaryMap);
			double duration = ((double)getTickCount() - start) / getTickFrequency();
			cout << "computeBinaryMap cost time is: " << duration * 1000 << "ms" << endl;

			imshow("Saliency Map", saliencyMap);
			imshow("Original Image", image);
			imshow("Binary Map", binaryMap);

			saliencyMap.convertTo(saliencyMap, CV_8UC3, 256);	// 转换格式才能保存图片
			imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap);
			imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap);
			waitKey(0);
		}
	}

	// ObjectnessBING
	else if (saliency_algorithm.find("BING") == 0)
	{
		// 判断模型是否存在
		if (training_path.empty())
		{
			cout << "Path of trained files missing! " << endl;
			return -1;
		}

		else
		{
			saliencyAlgorithm = ObjectnessBING::create();
			vector<Vec4i> saliencyMap;
			
			saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setTrainingPath(training_path);	// 提取模型文件参数
			saliencyAlgorithm.dynamicCast<ObjectnessBING>()->setBBResDir("Results");	// 将算法检测结果保存在Results文件夹内

			double start = static_cast<double>(getTickCount());	// 计算显著性
			bool success = saliencyAlgorithm->computeSaliency(image, saliencyMap);
			double duration = ((double)getTickCount() - start) / getTickFrequency();
			cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;

			if (success)
			{
				int ndet = int(saliencyMap.size());	// saliencyMap获取检测到的目标个数
				std::cout << "Objectness done " << ndet << std::endl;
				// 目标按可能性从大到小排列,maxd为显示前5个目标,step设置颜色,jitter设置矩形框微调
				int maxd = 5, step = 255 / maxd, jitter = 9;
				Mat draw = image.clone();
				for (int i = 0; i < std::min(maxd, ndet); i++)
				{
					Vec4i bb = saliencyMap[i];	// 获得矩形框坐标点
					Scalar col = Scalar(((i*step) % 255), 50, 255 - ((i*step) % 255));	// 设定颜色
					Point off(theRNG().uniform(-jitter, jitter), theRNG().uniform(-jitter, jitter));	// 矩形框微调
					rectangle(draw, Point(bb[0] + off.x, bb[1] + off.y), Point(bb[2] + off.x, bb[3] + off.y), col, 2);	// 画矩形
					rectangle(draw, Rect(20, 20 + i * 10, 10, 10), col, -1);	// 颜色标注
				}
				imshow("BING", draw);

				imwrite("Results/BING_draw.jpg", draw);
				waitKey();
			}
			else
			{
				std::cout << "No saliency found for " << video_name << std::endl;
			}
		}
	}

	// BinWangApr2014
	else if (saliency_algorithm.find("BinWangApr2014") == 0)
	{
		saliencyAlgorithm = MotionSaliencyBinWangApr2014::create();
		saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->setImagesize(image.cols, image.rows);	// 设置数据结构大小
		saliencyAlgorithm.dynamicCast<MotionSaliencyBinWangApr2014>()->init();	// 初始化

		bool paused = false;
		for (;; )
		{
			if (!paused)
			{
				cap >> frame;
				if (frame.empty())
				{
					return 0;
				}
				cvtColor(frame, frame, COLOR_BGR2GRAY);

				Mat saliencyMap;
				double start = static_cast<double>(getTickCount());
				saliencyAlgorithm->computeSaliency(frame, saliencyMap);
				double duration = ((double)getTickCount() - start) / getTickFrequency();
				cout << "computeSaliency cost time is: " << duration * 1000 << "ms" << endl;

				imshow("image", frame);
				imshow("saliencyMap", saliencyMap * 255);
			}

			char c = (char)waitKey(2);
			if (c == 'q')
				break;
			if (c == 'p')
				paused = !paused;
		}
	}
	destroyAllWindows();
	return 0;
}

python

# -*- coding: utf-8 -*-
import cv2
import random

def main():
    # 显著性检测算法
    # 可选:SPECTRAL_RESIDUAL,FINE_GRAINED,BING,BinWangApr2014
    saliency_algorithm = "FINE_GRAINED"
    video_name = "video/vtest.avi"
    # video_name = "video/dog.jpg";
    start_frame = 0		# 起始帧
    training_path = "ObjectnessTrainedModel"	# 模型路径

    # 如果算法名和视频名为空,停止检测
    if saliency_algorithm is None or video_name is None:
        print("Please set saliency_algorithm and video_name")
        return
        
    cap = cv2.VideoCapture(video_name)
    cap.set(cv2.CAP_PROP_POS_FRAMES, start_frame)	# 设置视频起始帧

    _, frame = cap.read()
    if frame is None:
        print("Please set saliency_algorithm and video_name")
        return

    image = frame.copy()

    # 根据输入的方法确定检测类型
    if saliency_algorithm.find("SPECTRAL_RESIDUAL") == 0:
        # 检测结果,白色区域表示显著区域
        saliencyAlgorithm = cv2.saliency.StaticSaliencySpectralResidual_create()
        start = cv2.getTickCount()	# 计算显著性
        success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
        duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
        print("computeBinaryMap cost time is: {} ms".format(duration * 1000))

        if success:
            start = cv2.getTickCount()	# 二值化图像
            _, binaryMap = saliencyAlgorithm.computeBinaryMap(saliencyMap)
            duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
            print("computeBinaryMap cost time is: {} ms".format(duration * 1000))

            cv2.imshow("Saliency Map", saliencyMap)
            cv2.imshow("Original Image", image)
            cv2.imshow("Binary Map", binaryMap)

            saliencyMap = (saliencyMap * 255)	# 转换格式才能保存图片
            cv2.imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap)
            cv2.imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap)
            cv2.waitKey(0)

    # FINE_GRAINED
    elif saliency_algorithm.find("FINE_GRAINED") == 0:
        saliencyAlgorithm = cv2.saliency.StaticSaliencyFineGrained_create()
        start = cv2.getTickCount()
        success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
        duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
        print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
        if success:
            start = cv2.getTickCount()
            _, binaryMap = saliencyAlgorithm.computeBinaryMap(saliencyMap)
            duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
            print("computeBinaryMap cost time is: {} ms".format(duration * 1000))

            cv2.imshow("Saliency Map", saliencyMap)
            cv2.imshow("Original Image", image)
            cv2.imshow("Binary Map", binaryMap)

            saliencyMap = (saliencyMap * 255)
            cv2.imwrite("Results/FINE_GRAINED_saliencyMap.jpg", saliencyMap)
            cv2.imwrite("Results/FINE_GRAINED_binaryMap.jpg", binaryMap)
            cv2.waitKey(0)

    elif saliency_algorithm.find("BING") == 0:
        # 判断模型是否存在
        if training_path is None:
            print("Path of trained files missing! ")
            return
        else:
            saliencyAlgorithm = cv2.saliency.ObjectnessBING_create()
            saliencyAlgorithm.setTrainingPath(training_path)	# 提取模型文件参数
            saliencyAlgorithm.setBBResDir("Results")	# 将算法检测结果保存在Results文件夹内
            
            start = cv2.getTickCount()
            success, saliencyMap = saliencyAlgorithm.computeSaliency(image)
            duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
            print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
            if success:
                ndet = saliencyMap.shape[0]	 # saliencyMap 获取检测到的目标个数
                print("Objectness done ", ndet)
                
                # 目标按可能性从大到小排列,maxd为显示前5个目标,step设置颜色,jitter设置矩形框微调
                maxd = 5
                step = 255 / maxd
                jitter = 9
                draw = image.copy()

                for i in range(0, min(maxd, ndet)):
                    bb = saliencyMap[i][0]
                    col = ((i * step) % 255), 50, 255 - ((i * step) % 255)
                    off = random.randint(-jitter, jitter), random.randint(-jitter, jitter)
                    cv2.rectangle(draw, (bb[0] + off[0], bb[1] + off[1]), (bb[2] + off[0], bb[3] + off[1]), col, 2)
                    cv2.rectangle(draw, (20, 20 + i * 10, 10, 10), col, -1)

                cv2.imwrite("Results/BING_draw.jpg", draw)
                cv2.imshow("BING", draw)
                cv2.waitKey(0)

    # 需要传入图像建模
    elif saliency_algorithm.find("BinWangApr2014") == 0:
        saliencyAlgorithm = cv2.saliency.MotionSaliencyBinWangApr2014_create()
        saliencyAlgorithm.setImagesize(image.shape[1], image.shape[0])	# 设置数据结构大小
        saliencyAlgorithm.init()	#  初始化
        paused = False

        while True:
            if not paused:
                _, frame = cap.read()
                if frame is None:
                    break

                frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
                start = cv2.getTickCount()
                success, saliencyMap = saliencyAlgorithm.computeSaliency(frame)
                duration = (cv2.getTickCount() - start) / cv2.getTickFrequency()
                print("computeBinaryMap cost time is: {} ms".format(duration * 1000))
                cv2.imshow("image", frame)
                cv2.imshow("saliencyMap", saliencyMap * 255)

            c = cv2.waitKey(2)
            c = chr(c) if c != -1 else 0
            if c == 'q':
                break
            if c == 'p':
                paused = not paused

    cv2.destroyAllWindows()
    return

if __name__ == '__main__':
    main()

参考链接

  1. https://blog.csdn.net/LuohenYJ/article/details/108607926
  • 5
    点赞
  • 13
    收藏
    觉得还不错? 一键收藏
  • 打赏
    打赏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包

打赏作者

libo-coder

你的鼓励将是我创作的最大动力

¥1 ¥2 ¥4 ¥6 ¥10 ¥20
扫码支付:¥1
获取中
扫码支付

您的余额不足,请更换扫码支付或充值

打赏作者

实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值