使用opencv和aircv 做图像位置匹配

本文提供两种方法做图像位置匹配:  模板匹配和特征匹配, 都使用opencv开发

需求: 给一张小图查找其在大图中的位置

需求图片:

 先说结论: 模板匹配只适合有缩放无旋转得,  特征匹配适合特征点多,图像旋转缩放都可看效果图:

需要注意: airac不能直接用,  有bug

图片中蓝色是模板匹配,红色特征匹配 

另外aircv其中有个函数基本无法通过, 最终将aircv代码中以下修改:

改成:

完整得py脚本如下:

# Name: contain_image.py
# Description: small image is in big image
# Author: CHIJING 20210909
# -*- coding: utf-8 -*-
import numpy as np
import pyautogui
import cv2
import aircv as ac
#import opencv-python
#import opencvpyxl
# 可以通过图片，在屏幕上定位图像所在的位置
# 找到返回的是一个4边距元组 (top, left, width, height)，没有找到返回None
# 全屏幕搜素
def _sift_instance(edge_threshold=100):
    if hasattr(cv2, 'SIFT'):
        return cv2.SIFT_create(edgeThreshold=edge_threshold)
    return cv2.xfeatures2d.SIFT_create(edgeThreshold=edge_threshold)


def sift_count(img):
    sift = _sift_instance()
    kp, des = sift.detectAndCompute(img, None)
    return len(kp)

FLANN_INDEX_KDTREE = 0

def find_all_sift(im_source, im_search, min_match_count=4, maxcnt=0):
    '''
    使用sift算法进行多个相同元素的查找
    Args:
        im_source(string): 图像、素材
        im_search(string): 需要查找的图片
        threshold: 阈值，当相识度小于该阈值的时候，就忽略掉
        maxcnt: 限制匹配的数量

    Returns:
        A tuple of found [(point, rectangle), ...]
        A tuple of found [{"point": point, "rectangle": rectangle, "confidence": 0.76}, ...]
        rectangle is a 4 points list
    '''
    sift = _sift_instance()
    flann = cv2.FlannBasedMatcher({'algorithm': FLANN_INDEX_KDTREE, 'trees': 5}, dict(checks=50))

    kp_sch, des_sch = sift.detectAndCompute(im_search, None)
    if len(kp_sch) < min_match_count:
        return None

    kp_src, des_src = sift.detectAndCompute(im_source, None)
    if len(kp_src) < min_match_count:
        return None

    h, w = im_search.shape[1:]

    result = []
    while True:
        # 匹配两个图片中的特征点，k=2表示每个特征点取2个最匹配的点
        matches = flann.knnMatch(des_sch, des_src, k=2)
        good = []
        for m, n in matches:
            # 剔除掉跟第二匹配太接近的特征点
            if m.distance < 0.9 * n.distance:
                good.append(m)

        if len(good) < min_match_count:
            break

        sch_pts = np.float32([kp_sch[m.queryIdx].pt for m in good]).reshape(-1, 1, 2)
        img_pts = np.float32([kp_src[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) 

        # M是转化矩阵
        M, mask = cv2.findHomography(sch_pts, img_pts, cv2.RANSAC, 5.0)
        matches_mask = mask.ravel().tolist()

        # 计算四个角矩阵变换后的坐标，也就是在大图中的坐标
        h, w = im_search.shape[:2]
        pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2)
        dst = cv2.perspectiveTransform(pts, M)

        # trans numpy arrary to python list
        # [(a, b), (a1, b1), ...]
        pypts = []
        for npt in dst.astype(int).tolist():
            pypts.append(tuple(npt[0]))

        lt, br = pypts[0], pypts[2]
        middle_point = (lt[0] + br[0]) / 2, (lt[1] + br[1]) / 2

        result.append(dict(result=middle_point,
            rectangle=pypts,
            confidence=(matches_mask.count(1), len(good)) #min(1.0 * matches_mask.count(1) / 10, 1.0)
        ))

        if maxcnt and len(result) >= maxcnt:
            break
        
        # 从特征点中删掉那些已经匹配过的, 用于寻找多个目标
        qindexes, tindexes = [], []
        for m in good:
            qindexes.append(m.queryIdx) # need to remove from kp_sch
            tindexes.append(m.trainIdx) # need to remove from kp_img

        def filter_index(indexes, arr):
            r = np.ndarray(0, np.float32)
            for i, item in enumerate(arr):
                if i not in qindexes:
                    r = np.append(r, item)
            return r
        kp_src = filter_index(tindexes, kp_src)
        des_src = filter_index(tindexes, des_src)

    return result
pass
#
#filename_result1 = 'D:\\pycode\\图像识别\\图片\\A组\\assert_ZhiTuShiTu.png'
#filename_screen1 = 'D:\\pycode\\图像识别\\图片\\A组\\2021-09-09_11_29_42assert.jpg'
filename_result1 = 'D:\\pycode\\assert_ZhiTuShiTu.png'
filename_screen1 = 'D:\\pycode\\a2021-09-09_11_29_42assert.jpg'
filename_result2 = 'D:\\pycode\\bat_inters_Point_Point.png'
filename_screen2 = 'D:\\pycode\\b2021-09-09_11_36_08assert.jpg'
filename_result3 = 'D:\\pycode\\c20181026153953158.png'
filename_screen3 = 'D:\\pycode\\c20181026153930243.png'

def GetBoxAndCenter(pts):
    circle_center_pos = (0,0)
    top_left =(0,0)
    right_bottom=(0,0)
    xmin=10000
    ymin=10000
    xmax=0
    ymax=0
    for m in pts:
        if m[0] <0 or m[1] < 0:
            continue;
        if m[0] < xmin:
            xmin =  m[0]    
        if m[0] > xmax:
            xmax =  m[0] 
        if m[1] < ymin:
            ymin =  m[1]    
        if m[1] > ymax:
            ymax =  m[1] 
    circle_center_pos = ((xmax-xmin)/2,(ymax-ymin)/2)
    top_left =(xmin,ymax)
    right_bottom=(xmax,ymin)
    return circle_center_pos,top_left,right_bottom,
pass

#找小图在大图中的位置,
def SearchsmallInBigPosition(filename_big, filename_small):
    im_result = cv2.imdecode(np.fromfile(filename_small,dtype=np.uint8),-1)  
    im_screen = cv2.imdecode(np.fromfile(filename_big,dtype=np.uint8),-1)  
    #smallimg = cv2.resize(im_result,(100,100))
    #由于png和jpg格式有差异这里rgb设True会更准确
    pos = ac.find_template(im_screen , im_result  , rgb=True, bgremove=True)
    #p2 = ac.find_sift( im_result,im_screen )

    if not pos:
        print('未找到')
        return None, None, None
    top_left = pos['rectangle'][0]  # 左上
    right_bottom = pos['rectangle'][3]  # 右下
    circle_center_pos = pos['result']  # 中心点坐标
    circle_center_pos = tuple(map(int, circle_center_pos))
    #print('准确率:', pos['confidence'])
    #print('中心坐标:',circle_center_pos)
    #print('左上右下:',top_left, right_bottom)
    return circle_center_pos,top_left,right_bottom
pass

def SearchsmallInBigPositionBySIFT(filename_big, filename_small):
    im_result = cv2.imdecode(np.fromfile(filename_small,dtype=np.uint8),-1)  
    im_screen = cv2.imdecode(np.fromfile(filename_big,dtype=np.uint8),-1)  
    pos = find_all_sift(im_screen,im_result,4,1)
    #im_result_gray = cv2.cvtColor( im_result, cv2.COLOR_RGB2GRAY )
    #im_screen_gray = cv2.cvtColor( im_screen, cv2.COLOR_RGB2GRAY )
    #pos = find_all_sift(im_screen_gray,im_result_gray,4,1 )
    if not pos:
        print('未找到')
        return None, None, None
    top_left = pos[0]['rectangle'][0]  # 左上
    right_bottom = pos[0]['rectangle'][3]  # 右下
    circle_center_pos = pos[0]['result']  # 中心点坐标
    circle_center_pos = tuple(map(int, circle_center_pos))
    #print('准确率:', pos['confidence'])
    #print('中心坐标:',circle_center_pos)
    #print('左上右下:',top_left, right_bottom)

    #这里opencv 求出来得是点集合,不能直接当作rectangle 用需要求外包盒重新求中点和外包矩形

    return GetBoxAndCenter(pos[0]['rectangle'])#circle_center_pos,top_left,right_bottom
pass

def draw_box(imsrc, pos, top_left, right_bottom, circle_radius,  color, line_width):
    """
    将小图在大图中圈出来
    """
    if not pos:
        print('match not found')
        return

    # 框出圆形
    # cv2.circle(img, pos, circle_radius, color, line_width)
    # 框出矩形
    cv2.rectangle(imsrc, top_left, right_bottom, color, line_width)  # 左上，右下
    cv2.namedWindow("objDetect", 0)  # 创建窗口时候可以鼠标随意拖动窗口改变大小，CV_WINDOW_NORMAL就是0
    # cv2.resizeWindow("objDetect", 640, 480) # 设置长宽大小为640*480
                                       # cv2.moveWindow("objDetect", 0, 0) # 移动窗口到（0,0）坐标
    cv2.imshow('objDetect', imsrc)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
pass



def main():
    
    circle_radius = 40
    color = (0, 0, 255)
    line_width = 3
    # draw circle


    circle_center_pos,top_left,right_bottom = SearchsmallInBigPosition(filename_screen1, filename_result1)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('模板匹配方法左上右下:',top_left, right_bottom)
    imsrc = cv2.imdecode(np.fromfile(filename_screen1,dtype=np.uint8),-1)
    color = (255, 0, 0)
    line_width = 3
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)

    circle_center_pos,top_left,right_bottom = SearchsmallInBigPositionBySIFT(filename_screen1, filename_result1)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)
    color = (0, 0, 255)
    line_width = 5
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)

    circle_center_pos,top_left,right_bottom = SearchsmallInBigPosition(filename_screen2, filename_result2)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('模板匹配方法左上右下:',top_left, right_bottom)
    imsrc = cv2.imdecode(np.fromfile(filename_screen2,dtype=np.uint8),-1)  
    color = (255, 0, 0)
    line_width = 3
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)
    circle_center_pos,top_left,right_bottom = SearchsmallInBigPositionBySIFT(filename_screen2, filename_result2)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)
    color = (0, 0, 255)
    line_width = 5
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)


    circle_center_pos,top_left,right_bottom = SearchsmallInBigPosition(filename_screen3, filename_result3)
    print('模板匹配方法中心坐标:',circle_center_pos) 
    print('模板匹配方法左上右下:',top_left, right_bottom)
    imsrc = cv2.imdecode(np.fromfile(filename_screen3,dtype=np.uint8),-1)  
    color = (255, 0, 0)
    line_width = 3
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)
    circle_center_pos,top_left,right_bottom = SearchsmallInBigPositionBySIFT(filename_screen3, filename_result3)
    print('特征点匹配中心坐标:',circle_center_pos) 
    print('特征点匹配左上右下:',top_left, right_bottom)
    color = (0, 0, 255)
    line_width = 5
    draw_box(imsrc, circle_center_pos, top_left, right_bottom, circle_radius, color, line_width)
pass

main()