全景图片拼接（python&&C++）

终于有空余时间来整理之前图片拼接的代码，要实现图片拼接的原因是课题需要。

需要实现对高分辨率（每个的图片大小大概为5500*4000）的图片拼接。尝试了许多方法，由于图片本身的特殊性，始终无法达到理想目标，但也有了还算不错的拼接结果。

这篇博客主要整理尝试的不同方法，如果能帮助到有需要的人就更好了。

一、opencv官方API&c++

关于环境配置的过程，可见下面的链接博客：

https://blog.csdn.net/packdge_black/article/details/107843346

样例和代码如下：

#include "opencv2/imgcodecs.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/stitching.hpp"
#include <opencv2/core.hpp>
#include <iostream>
#include <fstream>

using namespace std;
using namespace cv;

bool try_use_gpu = false;                  //默认不使用GPU
vector<Mat> imgs;
string result_name = "result.jpg";

int main(int argc, char* argv[])
{
	/**/
	Mat img1 = imread("7.jpg");
	Mat img2 = imread("8.jpg");

	Mat img1_1;
	Mat img2_2;

	/**/
	resize(img1, img1_1, Size(400, 300), 0, 0, CV_INTER_LINEAR);
	resize(img2, img2_2, Size(400, 300), 0, 0, CV_INTER_LINEAR);

	//resize(img3, img3_3, Size(400, 300), 0, 0, CV_INTER_LINEAR);
	//resize(img4, img4_4, Size(400, 300), 0, 0, CV_INTER_LINEAR);
	//resize(img5, img5_5, Size(400, 300), 0, 0, CV_INTER_LINEAR);

	/**/
	imgs.push_back(img1_1);
	imgs.push_back(img2_2);


	Stitcher stitcher = Stitcher::createDefault(try_use_gpu);

	Mat pano;
	Stitcher::Status status = stitcher.stitch(imgs, pano);


	if (status != Stitcher::OK)
	{
		cout << "Can't stitch images, error code = " << status << endl;
		return -1;
	}

	namedWindow(result_name);
	imshow(result_name, pano);
	//imwrite(result_name, pano);

	waitKey();
	//getchar();
	return 0;
}

样例图片：

 

拼接结果：

具体的官方文档链接：

https://docs.opencv.org/master/d8/d19/tutorial_stitcher.html

说明：

最重要的代码部分是：

第二行和第三行是最重要的两行，大多数的内容都封装进去了：

Mat pano;

Ptr<Stitcher> stitcher = Stitcher::create(mode);

Stitcher::Status status = stitcher->stitch(imgs, pano);

if (status != Stitcher::OK)

{

cout << "Can't stitch images, error code = " << int(status) << endl;

return EXIT_FAILURE;

}

 

 

 

主要的管线图（如果想了解更多的细节，可以使用C ++或python中可用的stitching_detailed源代码）

 

二、RANSAC算法+SIFT&Python

代码如下：

import cv2
import numpy as np
import sys

class Image_Stitching():
    def __init__(self) :
        self.ratio=0.85
        self.min_match=10
        self.sift=cv2.xfeatures2d.SIFT_create()
        self.smoothing_window_size=800

    def registration(self,img1,img2):
        kp1, des1 = self.sift.detectAndCompute(img1, None)
        kp2, des2 = self.sift.detectAndCompute(img2, None)
        matcher = cv2.BFMatcher()
        raw_matches = matcher.knnMatch(des1, des2, k=2)
        good_points = []
        good_matches=[]
        for m1, m2 in raw_matches:
            if m1.distance < self.ratio * m2.distance:
                good_points.append((m1.trainIdx, m1.queryIdx))
                good_matches.append([m1])
        img3 = cv2.drawMatchesKnn(img1, kp1, img2, kp2, good_matches, None, flags=2)
        cv2.imwrite('matching.jpg', img3)
        if len(good_points) > self.min_match:
            image1_kp = np.float32(
                [kp1[i].pt for (_, i) in good_points])
            image2_kp = np.float32(
                [kp2[i].pt for (i, _) in good_points])
            H, status = cv2.findHomography(image2_kp, image1_kp, cv2.RANSAC,5.0)
        return H

    def create_mask(self,img1,img2,version):
        height_img1 = img1.shape[0]
        width_img1 = img1.shape[1]
        width_img2 = img2.shape[1]
        height_panorama = height_img1
        width_panorama = width_img1 +width_img2
        offset = int(self.smoothing_window_size / 2)
        barrier = img1.shape[1] - int(self.smoothing_window_size / 2)
        mask = np.zeros((height_panorama, width_panorama))
        if version== 'left_image':
            mask[:, barrier - offset:barrier + offset ] = np.tile(np.linspace(1, 0, 2 * offset ).T, (height_panorama, 1))
            mask[:, :barrier - offset] = 1
        else:
            mask[:, barrier - offset :barrier + offset ] = np.tile(np.linspace(0, 1, 2 * offset ).T, (height_panorama, 1))
            mask[:, barrier + offset:] = 1
        return cv2.merge([mask, mask, mask])

    def blending(self,img1,img2):
        H = self.registration(img1,img2)
        height_img1 = img1.shape[0]
        width_img1 = img1.shape[1]
        width_img2 = img2.shape[1]
        height_panorama = height_img1
        width_panorama = width_img1 +width_img2

        panorama1 = np.zeros((height_panorama, width_panorama, 3))
        mask1 = self.create_mask(img1,img2,version='left_image')
        panorama1[0:img1.shape[0], 0:img1.shape[1], :] = img1
        panorama1 *= mask1
        mask2 = self.create_mask(img1,img2,version='right_image')
        panorama2 = cv2.warpPerspective(img2, H, (width_panorama, height_panorama))*mask2
        result=panorama1+panorama2

        rows, cols = np.where(result[:, :, 0] != 0)
        min_row, max_row = min(rows), max(rows) + 1
        min_col, max_col = min(cols), max(cols) + 1
        final_result = result[min_row:max_row, min_col:max_col, :]
        return final_result
def main(argv1,argv2):
    img1 = cv2.imread(argv1)
    img2 = cv2.imread(argv2)
    final=Image_Stitching().blending(img1,img2)
    cv2.imwrite('example.jpg', final)
if __name__ == '__main__':
    try: 
        main("C:/Users/10937/Desktop/Image-Stitching-OpenCV-master//1.jpg","C:/Users/10937/Desktop/Image-Stitching-OpenCV-master//2.jpg")
    except IndexError:
        print ("Please input two source images: ")
        print ("For example: python Image_Stitching.py '/Users/linrl3/Desktop/picture/p1.jpg' '/Users/linrl3/Desktop/picture/p2.jpg'")
    

样例图片：

过程：

先使用Ransac等鲁棒技术来提取特征，然后将它们进行匹配。

 

有许多不同的特征检测方法：

哈里斯角落探测器
缩放不变特征变换（SIFT），加速鲁棒特征（SURF），定向快速旋转BRIM（ORB）等。

matching image:

Final result:

额外尝试的代码：

import timeit

import cv2
import numpy as np


class Matcher:
    def __init__(self):
        self.surf = cv2.xfeatures2d.SURF_create()
        index_params = dict(algorithm=0, trees=5)
        search_params = dict(checks=50)
        self.flann = cv2.FlannBasedMatcher(index_params, search_params)

    def match(self, i1, i2):
        image_set_1 = self.get_SURF_features(i1)
        image_set_2 = self.get_SURF_features(i2)
        matches = self.flann.knnMatch(image_set_2["des"], image_set_1["des"], k=2)
        good = []
        for i, (m, n) in enumerate(matches):
            if m.distance < 0.7 * n.distance:
                good.append((m.trainIdx, m.queryIdx))

        if len(good) > 4:
            points_current = image_set_2["kp"]
            points_previous = image_set_1["kp"]

            matched_points_current = np.float32(
                [points_current[i].pt for (__, i) in good]
            )
            matched_points_prev = np.float32(
                [points_previous[i].pt for (i, __) in good]
            )

            H, _ = cv2.findHomography(
                matched_points_current, matched_points_prev, cv2.RANSAC, 4
            )
            return H
        return None

    def get_SURF_features(self, im):
        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        kp, des = self.surf.detectAndCompute(gray, None)
        return {"kp": kp, "des": des}


class Stitcher:
    def __init__(
        self,
        number_of_images,
        crop_x_min=None,
        crop_x_max=None,
        crop_y_min=None,
        crop_y_max=None,
    ):

        self.matcher_obj = Matcher()
        self.homography_cache = {}
        self.overlay_cache = {}

        self.count = number_of_images

        self.crop_x_min = crop_x_min
        self.crop_x_max = crop_x_max
        self.crop_y_min = crop_y_min
        self.crop_y_max = crop_y_max

    def stitch(self, images=[]):
        """
        stitches the images into a panorama
        """
        self.images = images

        self.prepare_lists()

        # left stitching
        start = timeit.default_timer()
        self.left_shift()
        self.right_shift()
        stop = timeit.default_timer()
        duration = stop - start
        print("stitching took %.2f seconds." % duration)

        if self.crop_x_min and self.crop_x_max and self.crop_y_min and self.crop_y_max:
            return self.result[
                self.crop_y_min : self.crop_y_max, self.crop_x_min : self.crop_x_max
            ]
        else:
            return self.result

    def prepare_lists(self):

        # reset lists
        self.left_list = []
        self.right_list = []

        self.center_index = int(self.count / 2)

        self.result = self.images[self.center_index]

        for i in range(self.count):
            if i <= self.center_index:
                self.left_list.append(self.images[i])
            else:
                self.right_list.append(self.images[i])

    def get_homography(self, image_1, image_1_key, image_2, image_2_key, direction):
        # TODO: use image indexes from the input array
        """
        Calculate the homography matrix between two images.
        Return from cache if possible.

        Args:
            image_1 (np.array) - first image
            image_1_key (str) - identifier for cache
            image_2 (np.array) - second image
            image_2_key (str) - identifier for cache
            direction (str) - "left" or "right"
        Returns:
            homography (np.array) - Homograpy Matrix
        """

        cache_key = "_".join([image_1_key, image_2_key, direction])
        homography = self.homography_cache.get(cache_key, None)
        if homography is None:
            # TODO: is the homography the same regardless of order??
            homography = self.matcher_obj.match(image_1, image_2)
            # put in cache
            self.homography_cache[cache_key] = homography
        return homography

    def left_shift(self):
        """
        stitch images center to left
        """
        # start off with center image
        a = self.left_list[0]

        for i, image in enumerate(self.left_list[1:]):
            H = self.get_homography(a, str(i), image, str(i + 1), "left")

            # inverse homography
            XH = np.linalg.inv(H)

            ds = np.dot(XH, np.array([a.shape[1], a.shape[0], 1]))
            ds = ds / ds[-1]

            f1 = np.dot(XH, np.array([0, 0, 1]))
            f1 = f1 / f1[-1]

            XH[0][-1] += abs(f1[0])
            XH[1][-1] += abs(f1[1])

            ds = np.dot(XH, np.array([a.shape[1], a.shape[0], 1]))
            offsety = abs(int(f1[1]))
            offsetx = abs(int(f1[0]))

            # dimension of warped image
            dsize = (int(ds[0]) + offsetx, int(ds[1]) + offsety)

            tmp = cv2.warpPerspective(a, XH, dsize, borderMode=cv2.BORDER_TRANSPARENT)

            # punch the image in there
            tmp[
                offsety : image.shape[0] + offsety, offsetx : image.shape[1] + offsetx
            ] = image

            a = tmp

        self.result = tmp

    def right_shift(self):
        """
        stitch images center to right
        """
        for i, imageRight in enumerate(self.right_list):
            imageLeft = self.result

            H = self.get_homography(imageLeft, str(i), imageRight, str(i + 1), "right")

            # args: original_image, matrix, output shape (width, height)
            result = cv2.warpPerspective(
                imageRight,
                H,
                (imageLeft.shape[1] + imageRight.shape[1], imageLeft.shape[0]),
                borderMode=cv2.BORDER_TRANSPARENT,
            )

            mask = np.zeros((result.shape[0], result.shape[1], 3), dtype="uint8")
            mask[0 : imageLeft.shape[0], 0 : imageLeft.shape[1]] = imageLeft
            self.result = self.blend_images(mask, result, str(i))

    def blend_images(self, background, foreground, i):
        """
        inspired by this answer:

        https://stackoverflow.com/a/54129424/1909378
        """

        only_right = self.overlay_cache.get(i, None)
        if only_right is None:
            only_right = np.nonzero(
                (np.sum(foreground, 2) != 0) * (np.sum(background, 2) == 0)
            )
            self.overlay_cache[i] = only_right

        background[only_right] = foreground[only_right]
        return background


if __name__ == "__main__":
    FRAME_WIDTH = 768
    FRAME_HEIGHT = 432

    shanghai_files = [
        "images2/1.jpg",
        "images2/2.jpg",
        "images2/3.jpg",
        "images2/4.jpg",
       # "images2/shanghai-05.png",
    ]

    shanghai = [
        cv2.resize(cv2.imread(f), (FRAME_WIDTH, FRAME_HEIGHT)) for f in shanghai_files
    ]

    crop_x_min = 30
    crop_x_max = 1764
    crop_y_min = 37
    crop_y_max = 471

    s = Stitcher(
        len(shanghai_files),
        crop_x_min=crop_x_min,
        crop_x_max=crop_x_max,
        crop_y_min=crop_y_min,
        crop_y_max=crop_y_max,
    )

    panorama = s.stitch(shanghai)

    cv2.imwrite("panorama.png", panorama)