OpenCV图像 OSG模型 vs2015 摄像头 图像 插入模型

本文参考了Github中的这个项目，基本就是把原作者的代码搬到了VS2015上，非常感谢原作者的工作。效果就是在摄像头采集到的图像中渲染模型然后显示，先看我的效果图：

环境：win10, vs2015, opencv3.4.13, OSG3.6.5

新建一个空白的Win32控制台应用程序，我的工程一共分为三个代码，如图：

各个部分的代码如下：

myhead.h

自定义头文件，声明两个类

#pragma once

#include <windows.h>
#include <osg/Camera>
#include <osg/PolygonMode>
#include <osg/Texture2D>
#include <osg/Geode>
#include <osg/Geometry>
#include <opencv2/opencv.hpp>

class VirtualCamera {
public:
	VirtualCamera(osg::ref_ptr<osg::Camera> cam);
	void updatePosition(double r, double p, double h, double x, double y, double z);

protected:
	double angle;
	osg::Matrix rotation;
	osg::Matrix translation;
	osg::ref_ptr<osg::Camera> camera;
};

class BackgroundCamera {
public:
	BackgroundCamera();
	void update(cv::Mat frame);
	osg::Geode* createCameraPlane(int textureWidth, int textureHeight);
	osg::Camera* createCamera(int textureWidth, int textureHeight);

protected:
	osg::ref_ptr<osg::Image> img;
};

classes.cpp

定义两个类中的各种函数

#include "myhead.h"

#include <stdio.h>
#include <iostream>

VirtualCamera::VirtualCamera(osg::ref_ptr<osg::Camera> cam)
{
	camera = cam;
	// Initial Values
	camera->setProjectionMatrixAsPerspective(40., 1., 1., 100.);
}

void VirtualCamera::updatePosition(double r, double p, double h, double x, double y, double z)
{
	osg::Matrixd myCameraMatrix;

	// Update Rotation
	rotation.makeRotate(
		osg::DegreesToRadians(r), osg::Vec3(0, 1, 0), // roll
		osg::DegreesToRadians(p), osg::Vec3(1, 0, 0), // pitch
		osg::DegreesToRadians(h), osg::Vec3(0, 0, 1)); // heading

													   // Update Translation
	translation.makeTranslate(x, y, z);
	myCameraMatrix = rotation * translation;
	osg::Matrixd i = myCameraMatrix.inverse(myCameraMatrix);
	camera->setViewMatrix(i*osg::Matrix::rotate(-(osg::PI_2), 1, 0, 0));
}



BackgroundCamera::BackgroundCamera() {
	// Create OSG Image from CV Mat
	img = new osg::Image;
}

void BackgroundCamera::update(cv::Mat frame)
{
	img->setImage(frame.cols, frame.rows, 3,
		GL_RGB, GL_BGR, GL_UNSIGNED_BYTE,
		(uchar*)(frame.data),
		osg::Image::AllocationMode::NO_DELETE, 1);
	img->dirty();
}

osg::Geode* BackgroundCamera::createCameraPlane(int textureWidth, int textureHeight)
{
	// CREATE PLANE TO DRAW TEXTURE
	osg::ref_ptr<osg::Geometry> quadGeometry = osg::createTexturedQuadGeometry(osg::Vec3(0.0f, 0.0f, 0.0f),
		osg::Vec3(textureWidth, 0.0f, 0.0f),
		osg::Vec3(0.0, textureHeight, 0.0),
		0.0f,
		1.0f,
		1.0f,
		0.0f);
	// PUT PLANE INTO NODE
	osg::ref_ptr<osg::Geode> quad = new osg::Geode;
	quad->addDrawable(quadGeometry);
	// DISABLE SHADOW / LIGHTNING EFFECTS
	int values = osg::StateAttribute::OFF | osg::StateAttribute::PROTECTED;
	quad->getOrCreateStateSet()->setAttribute(new osg::PolygonMode(osg::PolygonMode::FRONT_AND_BACK, osg::PolygonMode::FILL), values);
	quad->getOrCreateStateSet()->setMode(GL_LIGHTING, values);

	osg::ref_ptr<osg::Texture2D> texture = new osg::Texture2D;
	texture->setTextureSize(textureWidth, textureHeight);
	texture->setFilter(osg::Texture::MIN_FILTER, osg::Texture::LINEAR);
	texture->setFilter(osg::Texture::MAG_FILTER, osg::Texture::LINEAR);
	texture->setWrap(osg::Texture::WRAP_S, osg::Texture::REPEAT);
	texture->setWrap(osg::Texture::WRAP_T, osg::Texture::REPEAT);
	texture->setResizeNonPowerOfTwoHint(false);

	texture->setImage(img);

	// Apply texture to quad
	osg::ref_ptr<osg::StateSet> stateSet = quadGeometry->getOrCreateStateSet();
	stateSet->setTextureAttributeAndModes(0, texture, osg::StateAttribute::ON);

	return quad.release();
}

osg::Camera* BackgroundCamera::createCamera(int textureWidth, int textureHeight)
{
	osg::ref_ptr<osg::Geode> quad = createCameraPlane(textureWidth, textureHeight);
	//Bind texture to the quadGeometry, then use the following camera:
	osg::Camera* camera = new osg::Camera;
	// CAMERA SETUP
	camera->setReferenceFrame(osg::Camera::ABSOLUTE_RF);
	// use identity view matrix so that children do not get (view) transformed
	camera->setViewMatrix(osg::Matrix::identity());
	camera->setClearMask(GL_DEPTH_BUFFER_BIT);
	camera->setClearColor(osg::Vec4(0.f, 0.f, 0.f, 1.0));
	camera->setProjectionMatrixAsOrtho(0.f, textureWidth, 0.f, textureHeight, 1.0, 500.f);
	// set resize policy to fixed
	camera->setProjectionResizePolicy(osg::Camera::ProjectionResizePolicy::FIXED);
	// we don't want the camera to grab event focus from the viewers main camera(s).
	camera->setAllowEventFocus(false);
	// only clear the depth buffer
	camera->setClearMask(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
	//camera->setViewport( 0, 0, screenWidth, screenHeight );
	camera->setRenderOrder(osg::Camera::NESTED_RENDER);
	camera->addChild(quad);
	return camera;
}

OSG_test.cpp

主函数

#include <windows.h>
#include <osgViewer/Viewer>
#include <osgDB/ReadFile>
#include <osg/PositionAttitudeTransform>

#include "myhead.h"

#include "opencv2/highgui/highgui.hpp"
#include "opencv2/core/core.hpp"

int main(int argc, char** argv)
{
	int screenWidth, screenHeight, textureWidth, textureHeight;

	screenWidth = 640;
	screenHeight = 480;

	textureWidth = 640;
	textureHeight = 480;

	// OPENCV STUFF
	// OpenCV Webcam
	cv::VideoCapture cap(0);

	if (!cap.isOpened())
	{
		std::cout << "Webcam cannot open!\n";
		return 0;
	}

	// OSG STUFF
	// Create viewer
	osgViewer::Viewer viewer;
	viewer.setUpViewInWindow(50, 50, cap.get(CV_CAP_PROP_FRAME_WIDTH), cap.get(CV_CAP_PROP_FRAME_HEIGHT));

	// Main Camera
	osg::ref_ptr<osg::Camera>  camera = viewer.getCamera();
	VirtualCamera* vCamera = new VirtualCamera(camera);

	// Background-Camera (OpenCV Feed)
	BackgroundCamera bgCamera;
	osg::Camera* backgroundCamera = bgCamera.createCamera(textureWidth, textureHeight);

	// Load Truck Model as Example Scene
	osg::ref_ptr<osg::Node> truckModel = osgDB::readNodeFile("cessna.osg");
	osg::Group* truckGroup = new osg::Group();
	// Position of truck
	osg::PositionAttitudeTransform* position = new osg::PositionAttitudeTransform();

	truckGroup->addChild(position);
	position->addChild(truckModel);

	// Set Position of Model
	osg::Vec3 modelPosition(0, 80, 0);
	position->setPosition(modelPosition);

	// Create new group node
	osg::ref_ptr<osg::Group> group = new osg::Group;
	osg::Node* background = backgroundCamera;
	osg::Node* foreground = truckGroup;
	background->getOrCreateStateSet()->setRenderBinDetails(1, "RenderBin");
	foreground->getOrCreateStateSet()->setRenderBinDetails(2, "RenderBin");
	group->addChild(background);
	group->addChild(foreground);
	background->getOrCreateStateSet()->setMode(GL_DEPTH_TEST, osg::StateAttribute::OFF);
	foreground->getOrCreateStateSet()->setMode(GL_DEPTH_TEST, osg::StateAttribute::ON);

	// Add the groud to the viewer
	viewer.setSceneData(group.get());

	double angleRoll(0.);
	while (!viewer.done())
	{
		// Refresh Background Image
		cv::Mat frame;
		cap >> frame;
		bgCamera.update(frame);

		angleRoll += 0.5;
		// Update Virtual Camera (these Coordinates should be determined by some AR-Framework/Functionality)
		// They are just updated for demonstration purposes..
		// Position Parameters: Roll, Pitch, Heading, X, Y, Z
		vCamera->updatePosition(angleRoll, 0, 0, 0, 0, 0);
		//osg::notify(osg::WARN)<<"Angle: "<<  angleRoll <<std::endl;
		viewer.frame();
	}
	return 0;
}