scipy 优化求解warp参数

已知一对图像，分别是warp前后的图像，如何求解warp参数？

这里做个简单尝试

之前通过优化可以得到颜色转换系数，可以得到滤波核，那么是否也可以求解
几何变换的参数呢？
实验表示，不太行，或者说我的方法不太行。

实验的时候可以缩小图像，减少计算量

采用暴力搜索的方法求 仿射变换的旋转角度，位移等参数是可以的，但是速度很慢

完整代码如下：

import numpy as np
import cv2
import matplotlib.pyplot as plt
from scipy.optimize import least_squares

from warp_process import warp_affine, cal_sobel


def image_affine():
    file = r'match\20.png'
    image = cv2.imread(file)[..., ::-1]
    src = image
    # 设置3个对应点的坐标
    srcTri = np.array([[0, 0],
                       [src.shape[1] - 1, 0],
                       [0, src.shape[0] - 1]]).astype(np.float32)

    dstTri = np.array([[0, src.shape[1] * 0.33],
                       [src.shape[1] * 0.85, src.shape[0] * 0.25],
                       [src.shape[1] * 0.15, src.shape[0] * 0.7]]).astype(np.float32)
    # 求解旋转矩阵
    warp_mat = cv2.getAffineTransform(srcTri, dstTri)
    h, w, c = src.shape
    warp_mat = cv2.getRotationMatrix2D((w / 2, h / 2), 0, 0.5)
    # warp操作
    warp_dst = cv2.warpAffine(src, warp_mat, (src.shape[1], src.shape[0]))
    print(warp_mat)
    mask = np.sum(warp_dst, -1) > 0
    plt.imshow(warp_dst)
    plt.show()
    cv2.imwrite(file[:-4] + '_affine.png', warp_dst[..., ::-1])
    cv2.imwrite(file[:-4] + '_affine_mask.png', np.clip(mask * 255, 0, 255).astype(np.uint8))

def search(src,target, dzz, i, j, u, v):
    cent = (int(i), int(j))
    # print('center',cx.dtype, cy.dtype, cent, angle, scale)
    warp_mat = cv2.getRotationMatrix2D(cent, u, v)
    out = cv2.warpAffine(src, warp_mat, (src.shape[1] * dzz, src.shape[0] * dzz),
                         flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP)
    loss = np.mean((out - target)**2)
    # print('searchL', cx, cy, angle, scale, loss)
    return loss
if __name__ == "__main__":
    # image_affine()


    def model(warp_mat, src):
        h, w, c = src.shape
        # warp_mat = warp_mat.reshape(2, 3)
        center = (w // 2, h // 2)
        angle = warp_mat[0]
        scale = warp_mat[1]
        warp_mat = cv2.getRotationMatrix2D(center, angle, scale)
        out = cv2.warpAffine(src, warp_mat, (src.shape[1]*dzz, src.shape[0]*dzz), flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP)
        # out = warp_affine(src, warp_mat, h, w)
        return out

    # 定义函数, 不需要自己写损失函数，在调用函数的时候可以指定损失函数类型
    def min_fun_lq(warp_mat, src, target, mask):
        # t = cal_sobel(model(warp_mat, src)) - cal_sobel(target)
        t = model(warp_mat, src) - target
        # print(t.shape, mask.shape)
        out = t #* mask[..., None]
        return out.reshape(-1)  # 需要是1维数组


    # file1 = r'match\20.png'
    # file2 = r'match\20_affine.png'
    # mask_file = r'match\20_affine_mask.png'
    # image1 = cv2.imread(file1)[..., ::-1] / 255
    # image2 = cv2.imread(file2)[..., ::-1] / 255
    # mask = cv2.imread(mask_file, 0) > 0

    '1. 读取image1 和 生成image2'
    file1 = r'test.jpg'
    image1 = cv2.imread(file1)[..., ::-1] / 255
    h, w, c = image1.shape
    center = (w // 2, h // 2)
    angle = -50
    scale = 0.6

    affine_mat = cv2.getRotationMatrix2D(center, angle, scale)
    print(affine_mat)
    dzz = 4
    image2 = cv2.warpAffine(image1, affine_mat, (image1.shape[1]*dzz, image1.shape[0]*dzz), flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP)
    '2. 初始化转换矩阵 和 优化'
    # affine_mat = np.random.uniform(0, 1, [2, 3])
    # affine_mat = np.zeros([2, 3])
    # affine_mat[0, 0] = 0.4
    # affine_mat[1, 1] = 0.5
    # affine_mat = affine_mat.reshape(-1)
    #
    # affine_para = np.array([0, 1])
    # mask = np.ones([h, w], dtype=np.uint8)
    # print(mask.shape)
    # res = least_squares(min_fun_lq, affine_para, args=(image1, image2, mask), verbose=1)
    # # affine_mat = res.x.reshape(2, 3)
    # print('ret affine : ', res.x)
    # center = (w // 2, h // 2)
    # angle = res.x[0]
    # scale = res.x[1]
    # affine_mat = cv2.getRotationMatrix2D(center, angle, scale)

    '2. 暴力搜索'
    cx = np.arange(10, w-10, w//10)
    cy = np.arange(10, h-10, h//10)
    angle = np.arange(0, 160, 10) - 80
    scale = np.arange(0.2, 2, 0.2)
    print(cx, cy, angle, scale, cx.dtype, cy.dtype, angle.dtype, scale.dtype)

    loss_min = 10000000
    for i in cx:
        for j in cy:
            for u in angle:
                for v in scale:
                    loss = search(image1, image2, dzz, i, j, u, v)
                    if loss < loss_min:
                        loss_min = loss
                        print('searchL', i, j, u, v, loss)
                        ii, jj, uu, vv = i, j, u, v

    center = (ii // 2, jj // 2)
    angle = u
    scale = v
    affine_mat = cv2.getRotationMatrix2D(center, angle, scale)

    '3. apply affine'
    # affine_mat[0, 2] *= w
    # affine_mat[1, 2] *= h
    # warp操作
    warp_dst = cv2.warpAffine(image1, affine_mat, (image1.shape[1]*dzz, image1.shape[0]*dzz), flags=cv2.INTER_LINEAR + cv2.WARP_INVERSE_MAP)
    warp_dst = np.clip(warp_dst*255, 0, 255).astype(np.uint8)
    cv2.imwrite(file1[:-4] + '_affine_sci.png', warp_dst)

    plt.figure()
    plt.subplot(131)
    plt.imshow(image1)
    plt.subplot(132)
    plt.imshow(image2)
    plt.subplot(133)
    plt.imshow(warp_dst)
    plt.show()

后续尝试pytorch优化，光流等方法，看是否可以得到 warp参数

利用深度学习求解位姿变换矩阵的论文 或 求解光流的论文 都已经有不少。

torchgeometry 是一个基于pytorch的几何库，后续可能会用到

[1]https://zhuanlan.zhihu.com/p/349741938
[2]https://kornia.readthedocs.io/en/v0.1.2/_modules/torchgeometry/utils.html