智能老旧模糊照片修复——C++实现GFPGAN模型推理

前言

1.回家过年，总是有意或无意间翻到一些旧时的照片，旧照片作为时光记忆的载体和岁月流转的见证，不单单是过去美好时光的传承者，同时也是每个人的情结和怀念的寄托。随着时间的流逝,许多老照片都因为自然或人为原因，受到了侵蚀损坏，画面模糊、褪色、照片磨损严重等现象，甚至还有的因为保管不好导致照片面目全非。作为一个程序员，随着旧照片的损坏，你再也无法证明你也曾经满头秀发，你也曾经意气风发，有过明亮的眼眸。
2.GFPGAN是Tencent开源的一个旧照片修复算法，它能够让这些老照片恢复原有的光泽，使用了GAN算法对照片进行修复，效果比其他同类模型都有更好的表现。如果对算法和如何训练模型可转到github:https://github.com/TencentARC/GFPGAN .
3.这里只是演示如何使用C++调用官方给出训练好的模型，开发环境是win10， vs2019, opencv4.5, ncnn,如果要启用GPU加速，所以用到VulkanSDK，实现语言是C++。
4.官方给出的测试效果对比图：

C++代码实现：
第一张是原图，第二张是只修复人脸，第三张是修复全身：

二、实现代码

#include <vector>
#include <ostream>
#include <random>
#include <chrono>
#include <stdio.h>
#include <fstream>
#include <opencv2/opencv.hpp>
#include <ncnn/net.h>
#include <ncnn/cpu.h>
#include "gfpgan.h"
#include "face.h"
#include "realesrgan.h"


static void toOcv( const ncnn::Mat& result, cv::Mat& out)
{
    cv::Mat cv_result_32F = cv::Mat::zeros(cv::Size(512, 512), CV_32FC3);
    for (int i = 0; i < result.h; i++)
    {
        for (int j = 0; j < result.w; j++)
        {
            cv_result_32F.at<cv::Vec3f>(i, j)[2] = (result.channel(0)[i * result.w + j] + 1) / 2;
            cv_result_32F.at<cv::Vec3f>(i, j)[1] = (result.channel(1)[i * result.w + j] + 1) / 2;
            cv_result_32F.at<cv::Vec3f>(i, j)[0] = (result.channel(2)[i * result.w + j] + 1) / 2;
        }
    }

    cv::Mat cv_result_8U;
    cv_result_32F.convertTo(cv_result_8U, CV_8UC3, 255.0, 0);

    cv_result_8U.copyTo(out);

}

void  mergeImage(std::vector<cv::Mat>& src_vor, cv::Mat& cv_dst, int channel)
{
    cv::Mat img_merge;
    cv::Size size(src_vor.at(0).cols * src_vor.size(), src_vor.at(0).rows);
    if (channel == 1)
    {
        img_merge.create(size, CV_8UC1);
    }
    else if (channel == 3)
    {
        img_merge.create(size, CV_8UC3);
    }
    for (int i = 0; i < src_vor.size(); i++)
    {
        cv::Mat cv_temp = img_merge(cv::Rect(src_vor.at(i).cols * i, 0, src_vor.at(i).cols, src_vor.at(i).rows));
        src_vor.at(i).copyTo(cv_temp);
    }

    cv_dst = img_merge.clone();
}

static void pasteFacesInputImage(const cv::Mat& restored_face,cv::Mat& trans_matrix_inv,cv::Mat& bg_upsample)
{
    trans_matrix_inv.at<float>(0, 2) += 1.0;
    trans_matrix_inv.at<float>(1, 2) += 1.0;

    cv::Mat inv_restored;
    cv::warpAffine(restored_face, inv_restored, trans_matrix_inv, bg_upsample.size(), 1, 0);
    cv::Mat mask = cv::Mat::ones(cv::Size(512, 512), CV_8UC1) * 255;
    cv::Mat inv_mask;
    cv::warpAffine(mask, inv_mask, trans_matrix_inv, bg_upsample.size(), 1, 0);
    cv::Mat inv_mask_erosion;
    cv::Mat kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(4, 4));
    cv::erode(inv_mask, inv_mask_erosion, kernel);
    cv::Mat pasted_face;
    cv::bitwise_and(inv_restored, inv_restored, pasted_face, inv_mask_erosion);

    int total_face_area = cv::countNonZero(inv_mask_erosion);
    int w_edge = int(std::sqrt(total_face_area) / 20);
    int erosion_radius = w_edge * 2;
    cv::Mat inv_mask_center;
    kernel = cv::getStructuringElement(cv::MORPH_ELLIPSE, cv::Size(erosion_radius, erosion_radius));
    cv::erode(inv_mask_erosion, inv_mask_center, kernel);

    int blur_size = w_edge * 2;
    cv::Mat inv_soft_mask;
    cv::GaussianBlur(inv_mask_center, inv_soft_mask, cv::Size(blur_size + 1, blur_size + 1), 0, 0, 4);

    for (int h = 0; h < bg_upsample.rows; h++)
    {
        for (int w = 0; w < bg_upsample.cols; w++)
        {
            float alpha = inv_soft_mask.at<uchar>(h, w) / 255.0;
            bg_upsample.at<cv::Vec3b>(h, w)[0] = pasted_face.at<cv::Vec3b>(h, w)[0] * alpha + (1 - alpha) * bg_upsample.at<cv::Vec3b>(h, w)[0];
            bg_upsample.at<cv::Vec3b>(h, w)[1] = pasted_face.at<cv::Vec3b>(h, w)[1] * alpha + (1 - alpha) * bg_upsample.at<cv::Vec3b>(h, w)[1];
            bg_upsample.at<cv::Vec3b>(h, w)[2] = pasted_face.at<cv::Vec3b>(h, w)[2] * alpha + (1 - alpha) * bg_upsample.at<cv::Vec3b>(h, w)[2];
        }
    }
}

int main(int argc, char** argv)
{
    GFPGAN gfpgan;
    gfpgan.initModel("models/encoder.param", "models/encoder.bin", "models/style.bin");

    Face face_detector;
    face_detector.initModel("models/blazeface.param", "models/blazeface.bin");

    RealESRGAN real_esrgan;
    real_esrgan.initModel("models/real_esrgan.param", "models/real_esrgan.bin");

    std::string path = "images";
    std::vector<std::string> filenames;
    cv::glob(path, filenames, false);

    int i = 0;
    for (auto v : filenames)
    {
        cv::Mat img = cv::imread(v, 1);
        cv::Mat bg_upsample;
        real_esrgan.tile_process(img, bg_upsample);

        std::vector<cv::Mat> trans_img;
        std::vector<cv::Mat> trans_matrix_inv;
        std::vector<Object> objects;
        face_detector.detect(img, objects);
        face_detector.align_warp_face(img, objects, trans_matrix_inv, trans_img);

        for (size_t i = 0; i < objects.size(); i++)
        {
            ncnn::Mat gfpgan_result;
            gfpgan.process(trans_img[i], gfpgan_result);

            cv::Mat restored_face;
            toOcv(gfpgan_result, restored_face);

            pasteFacesInputImage(restored_face, trans_matrix_inv[i], bg_upsample);

        }

        ncnn::Mat gfpgan_result;
        gfpgan.process(img, gfpgan_result);

        cv::Mat restored_face;
        toOcv(gfpgan_result, restored_face);

        cv::resize(img, img, bg_upsample.size());
        cv::resize(restored_face, restored_face, bg_upsample.size());
        std::vector<cv::Mat> cv_dsts{ img, restored_face, bg_upsample };

        cv::Mat cv_dst;
        mergeImage(cv_dsts, cv_dst, 3);

        cv::imwrite(std::to_string(i)+".png", cv_dst);

        i++;
    }

    return 0;
}

效果图：

三.资源

1.带界面的可执行文件：https://download.csdn.net/download/matt45m/79972131
2.不带界面的小源码和模型：https://download.csdn.net/download/matt45m/79973523