大图滑动窗口式切片识别

Cpp

main.cpp

#include "slice_image.h"

int main(int argc, char** argv)
{
	const char* image_path = "E:\\Dataset\\pingtu\\s.jpg";
	cv::Mat img = cv::imread(image_path);
	int stride = 4000;
	cv::Scalar windowSize = cv::Scalar(5000, 5000);
	bool keepWindowSize = true;
	cv::Point offset(0, 0);
	std::vector<std::vector<cv::Mat>> patches;
	// 窗口大小为windowSize，按照步长为stride进行滑窗式切片
	si::sliceImage(img, patches, windowSize, stride, keepWindowSize, offset);
	
	std::vector<cv::Rect> result;
	si::inferPatches(patches, result, windowSize, stride, keepWindowSize, offset);
}

slice_image.h

#pragma once
#include "opencv2/core/core.hpp"  
#include "opencv2/highgui.hpp" 
#include "opencv2/imgproc.hpp"  

namespace si // sliceImage
{
	/***************************************************************
	  *  @brief                 图像滑动窗口式的切片，窗口大小为windowSize，按照步长为stride进行滑窗式切片
	  *  @param  src            输入原图像
	  *  @param  patches         切片后的图像集合
	  *  @param  windowSize     切片小图尺寸
	  *  @param  stride         滑动窗口的步长
	  *  @param  keepWindowSize 最右和最下边的图像不够windowSize大小的时候是否保持windowSize大小，如果保持的话会向左上偏移
	  *  @param  offset         keepWindowSize为true时向左上偏移量大小
	 **************************************************************/
	int sliceImage(const cv::Mat& src, std::vector<std::vector<cv::Mat>>& patches, const cv::Scalar& windowSize, const int& stride,
		const bool& keepWindowSize, cv::Point& offset);

	/***************************************************************
	  *  @brief                 对所有切片后的小图进行识别，然后将识别结果（坐标）映射回大图上
	  *  @param  patches        所有切片后的小图集合
	  *  @param  result         大图上的识别结果
	  *  @param  windowSize     切片小图尺寸
	  *  @param  stride         滑动窗口的步长
	  *  @param  keepWindowSize 最右和最下边的图像不够windowSize大小的时候是否保持windowSize大小，如果保持的话会向左上偏移
	  *  @param  offset         keepWindowSize为true时向左上偏移量大小
	 **************************************************************/
	int inferPatches(std::vector<std::vector<cv::Mat>>& patches, std::vector<cv::Rect>& result, const cv::Scalar& windowSize, const int& stride,
		const bool& keepWindowSize, cv::Point& offset);
}

inline std::vector<cv::Rect> inference(cv::Mat& patch)
{
	printf("Do some AI algorithms \n");
	std::vector<cv::Rect> r;
	return r;
}

inline void nms(std::vector<cv::Rect>& result)
{
	printf("nms algorithms \n");
}

slice_image.cpp

#include "slice_image.h"

int si::sliceImage(const cv::Mat& src, std::vector<std::vector<cv::Mat>>& patches, const cv::Scalar& windowSize,
	const int& stride, const bool& keepWindowSize, cv::Point& offset)
{
	if (src.empty())
	{
		printf("src is emppty...");
		return -1;
	}
	int windowWidth = windowSize[0];
	int windowHeight = windowSize[1];
	int srcHeight = src.rows;
	int srcWidth = src.cols;
	if (src.size() == cv::Size(windowWidth, windowHeight))
	{
		patches = { { src } };
		return 0;
	}
	for (int y = 0; y + windowHeight - stride< srcHeight; y += stride)
	{
		std::vector<cv::Mat> dstw;

		for (int x = 0; x + windowWidth - stride < srcWidth; x += stride)
		{
			cv::Rect patchRect;
			if (keepWindowSize)
			{
				int patchX = x;
				int patchY = y;
				if (patchX + windowWidth > srcWidth)
				{
					offset.x = patchX + windowWidth - srcWidth;
					patchX = srcWidth - windowWidth;
				}
				if (patchY + windowHeight > srcHeight)
				{
					offset.y = patchY + windowHeight - srcHeight;
					patchY = srcHeight - windowHeight;
				}
				patchRect = cv::Rect(patchX, patchY, windowWidth, windowHeight);
			}
			else
			{
				int patchW = windowSize[0];
				int patchH = windowSize[1];
				if (x + windowWidth > srcWidth)
				{
					offset.x = x + windowWidth - srcWidth;
					patchW = srcWidth - x;
				}
				if (y + windowHeight > srcHeight)
				{
					offset.y = y + windowHeight - srcHeight;
					patchH = srcHeight - y;
				}
				patchRect = cv::Rect(x, y, patchW, patchH);
			}

			cv::Mat patch(src, patchRect);
			dstw.push_back(patch);
		}
		patches.push_back(dstw);
	}

	return 1;
}

int si::inferPatches(std::vector<std::vector<cv::Mat>>& patches, std::vector<cv::Rect>& result, const cv::Scalar& windowSize, 
	const int& stride, const bool& keepWindowSize, cv::Point& offset)
{
	if (patches.size() == 0) { return -1; }
	int totalPatches = 0;
	for (const auto& innerVector : patches) {
		totalPatches += innerVector.size();
	}
	for (int h = 0; h < patches.size(); h++)
	{
		std::vector<cv::Mat> patchesW;
		cv::Mat patchW;
		for (int w = 0; w < patches[h].size(); w++)
		{
			cv::Mat patch = patches[h][w];
			if (!patch.data) { 
				printf("patch [%d][%d] is empty", w, h);
				return -1; 
			}

			// 1. 目标检测算法对切片后的小图进行识别
			//cv::namedWindow("patch", 0);
			//cv::imshow("patch", patch);
			//cv::waitKey(0);
			std::string filename = std::to_string(h) + "_" + std::to_string(w) + ".jpg";
			cv::imwrite(filename, patch);
			std::vector<cv::Rect> patchResult = inference(patch);

			int top_left_x = w * stride;  //左上角的坐标
			int top_left_y = h * stride;
			if (keepWindowSize)
			{

				if (h == patches.size() - 1)
				{
					top_left_y -= offset.y;
				}

				if (w == patches[h].size() - 1)
				{
					top_left_x -= offset.x;
				}

			}
			// 2. 将小图坐标映射会原图上
			for (int i = 0; i < patchResult.size(); i++)
			{
				patchResult[i].x += top_left_x;
				patchResult[i].y += top_left_y;
				result.push_back(patchResult[i]);
			}
		}
	}
	// 3. 在大图上在做一次nms
	nms(result);

	return 1;
}

Csharp

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

using OpenCvSharp;
using ImageSlicerInferSharp;

namespace ImageSlicerInferSharp
{
    // ImageSlicerInfer 大图滑动窗口式切片成多个小图，对小图做AI识别，返回识别结果
    public class ImageSlicerInfer
    {
        public ImageSlicerInfer() { }

        // SliceImage 图像滑动窗口式的切片，窗口大小为windowSize，按照步长为stride进行滑窗式切片
        public int SliceImage(Mat src, out List<List<Mat>> patches, Scalar windowSize, int stride, bool keepWindowSize, ref Point offset)
        {
            patches = new List<List<Mat>>();

            if (src.Empty())
            {
                Console.WriteLine("src is empty...");
                return -1;
            }

            int windowWidth = (int)windowSize.Val0;
            int windowHeight = (int)windowSize.Val1;
            int srcHeight = src.Rows;
            int srcWidth = src.Cols;

            if (src.Size() == new Size(windowWidth, windowHeight))
            {
                patches.Add(new List<Mat> { src });
                return 0;
            }

            for (int y = 0; y + windowHeight - stride < srcHeight; y += stride)
            {
                List<Mat> dstw = new List<Mat>();

                for (int x = 0; x + windowWidth - stride < srcWidth; x += stride)
                {
                    Rect patchRect;

                    if (keepWindowSize)
                    {
                        int patchX = x;
                        int patchY = y;

                        if (patchX + windowWidth > srcWidth)
                        {
                            offset.X = patchX + windowWidth - srcWidth;
                            patchX = srcWidth - windowWidth;
                        }

                        if (patchY + windowHeight > srcHeight)
                        {
                            offset.Y = patchY + windowHeight - srcHeight;
                            patchY = srcHeight - windowHeight;
                        }

                        patchRect = new Rect(patchX, patchY, windowWidth, windowHeight);
                    }
                    else
                    {
                        int patchW = (int)windowSize.Val0;
                        int patchH = (int)windowSize.Val1;

                        if (x + windowWidth > srcWidth)
                        {
                            offset.X = x + windowWidth - srcWidth;
                            patchW = srcWidth - x;
                        }

                        if (y + windowHeight > srcHeight)
                        {
                            offset.Y = y + windowHeight - srcHeight;
                            patchH = srcHeight - y;
                        }

                        patchRect = new Rect(x, y, patchW, patchH);
                    }

                    Mat patch = new Mat(src, patchRect);
                    dstw.Add(patch);
                }

                patches.Add(dstw);
            }

            return 1;
        }

        public struct DetObjects
        {
            public int label { get; set; }
            public Rect rect { get; set; }
            // 其他属性...
        }

        // 目标检测AI算法
        static List<DetObjects> Inference(Mat patch)
        {
            // 实现目标检测算法对图像进行推理，这里只是一个示例
            // 这里可以替换为您实际的目标检测算法
            List<DetObjects> detectedObjects = new List<DetObjects>();
            DetObjects detObjs = new DetObjects
            {
                label = 1, // 设置标签值
                rect = new Rect(10, 10, 100, 100) // 设置矩形区域的位置和大小// 其他属性...
                };
            // 模拟检测到一个对象
            detectedObjects.Add(detObjs);
            return detectedObjects;
        }

        static void Nms(List<DetObjects> result, float nms_threshold = 0.65f)
        {
            // 实现非极大值抑制算法,去除重叠的目标框
            // 这里只是一个示例
            result.Sort((b1, b2) => (b2.rect.Width * b2.rect.Height).CompareTo(b1.rect.Width * b1.rect.Height));
            for (int i = 0; i < result.Count; ++i)
            {
                for (int j = i + 1; j < result.Count;)
                {
                    int cls1 = result[i].label;
                    int cls2 = result[j].label;
                    if (cls1 != cls2)
                    {
                        continue;
                    }
                    Rect r1 = result[i].rect;
                    Rect r2 = result[j].rect;
                    Rect overR = Rect.Intersect(r1, r2);
                    float area1 = r1.Width * r1.Height;
                    float area2 = r2.Width * r2.Height;
                    float overArea = overR.Width * overR.Height;

                    if (area1 + area2 - overArea == 0)
                    {
                        result.RemoveAt(j);
                        continue;
                    }

                    float iou = overArea / (area1 + area2 - overArea);

                    if (cls1 == cls2)
                    {
                        float miniou = overArea / Math.Min(area1, area2);
                        if (miniou > 0.5)
                        {
                            result.RemoveAt(j);
                            continue;
                        }
                    }

                    if (iou >= nms_threshold)
                    {
                        result.RemoveAt(j);
                    }
                    else
                    {
                        ++j;
                    }
                }
            }
            Console.WriteLine("Performing non-maximum suppression");
        }

        public static int InferPatches(List<List<Mat>> patches, List<DetObjects> result, Scalar windowSize, int stride, bool keepWindowSize, ref Point offset)
        {
            if (patches.Count == 0) { return -1; }

            for (int row = 0; row < patches.Count; row++)
            {
                for (int col = 0; col < patches[row].Count; col++)
                {
                    Mat patch = patches[row][col];

                    if (patch.Empty())
                    {
                        Console.WriteLine($"Patch is empty");
                        return -1;
                    }

                    string filename = $"{row}_{col}.jpg";
                    Cv2.ImWrite(filename, patch);

                    // 1. 目标检测算法对切片后的小图进行识别
                    List<DetObjects> patchResult = Inference(patch);

                    // 2. 将小图坐标映射会原图上
                    int top_left_x = col * stride;
                    int top_left_y = row * stride;

                    if (keepWindowSize)
                    {
                        if (row == patches.Count - 1)
                        {
                            top_left_y -= offset.Y;
                        }
                        if (col == patches[row].Count - 1)
                        {
                            top_left_x -= offset.X;
                        }
                    }
                    for (int i = 0; i < patchResult.Count; i++)
                    {
                        DetObjects adjustedRect = new DetObjects
                        {
                            label = patchResult[i].label, // 设置标签值
                            rect = new Rect(patchResult[i].rect.X + top_left_x, patchResult[i].rect.Y + top_left_y,
                                            patchResult[i].rect.Width, patchResult[i].rect.Height)
                            };
                        result.Add(adjustedRect);
                    }
                }
            }
            // 3. 在大图上在做一次nms
            Nms(result);

            return 1;
        }
    }
}

namespace SliceImageCsharp
{
    internal class Program
    {
        static void Main(string[] args)
        {
            Mat src = Cv2.ImRead("E:\\Dataset\\pingtu\\s.jpg");

            // 参数通过UI界面填入
            List<List<Mat>> patches;
            Scalar windowSize = new Scalar(5000, 5000); // 设置窗口大小
            int stride = 4000; // 设置步长
            bool keepWindowSize = true; // 是否保持窗口大小
            Point offset = new Point(0, 0);

            ImageSlicerInfer imgSliceInfer = new ImageSlicerInfer();


            List<ImageSlicerInfer.DetObjects> detectResult = new List<ImageSlicerInfer.DetObjects>();
            int siFlag = imgSliceInfer.SliceImage(src, out patches, windowSize, stride, keepWindowSize, ref offset);

            if (siFlag != -1)
            {
                int ipFlag = ImageSlicerInfer.InferPatches(patches, detectResult, windowSize, stride, keepWindowSize, ref offset);
                if (ipFlag == -1) 
                {
                    Console.WriteLine($"patches is empty");
                }
            }
        }
    }
}