使用opencvsharp实现对Mask_Rcnn的调用（C#）

版权所有，翻版必究。

运行环境：WIN10，pycharm，相应的CUDA，CUDNN，tensorflow1.15.0，tensorflow-gpu-1.14.0，Anaconda3

https://blog.csdn.net/fightingxyz/article/details/107013155有空的先看看这篇文章，这边文章是在这个文章的基础上来做的。主要是前一篇文章是用来介绍怎么获取.pb文件和.pbtxt文件的。然后此篇博客是对此的改写，将其变成C#下的版本。如果有问题欢迎互相讨论，不惜勿喷！

第一步：获取.pb文件和.pbtxt文件（模型准备！）。详情请参见：https://blog.csdn.net/fightingxyz/article/details/107013155

主要测试的图像是这种类型：

上一篇文章中，有详细说明相关内容。主要是用来预测apple banana orange。我用C#的测试结果为：

第二步：配置C#的调用环境，看下图：

主要依赖库为这些！补充一句，必须要是opencv4.0及其以上才行，因为低版本还没有嵌入调用mask_rcnn的接口。

上面安装库中的TensorFlowSharp可用可不用，因为程序中没用(●'◡'●)

第三步：话不多说，详情看代码！

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using OpenCvSharp;
using TensorFlow;
using System.IO;

namespace test10
{
    class Program
    {
        static void Main(string[] args)
        {
            string textGraph = "mask_rcnn.pbtxt";
            string modelWeights = "frozen_inference_graph.pb";
            string classesFile = "mscoco_labels.names";
            string colorsFile = "colors.txt";

            float confThreshold = 0.9f;  //此处只是设置你的预测框阈值，判断是否显示

            float maskThreshold = 0.3f;


            List<string> classes = new List<string>();   //用来存放类型数据在其中
            List<Scalar> colors = new List<Scalar>();   //用来存放画图的颜色
            Mat blob;
            float m_fWidthScale;
            float m_fHeighScale;

            //下面这段主要是用于读取文件，并将其中内容放置classes中。
            //和上一篇文章中的void LoadLabelAndColor()函数一个意思，不懂可以去看看python版本或者C++版本
            FileStream myfs = new FileStream(classesFile, FileMode.Open);
            StreamReader reader = new StreamReader(myfs);
            string con;
            con = reader.ReadToEnd();
            var a = con.Split(new string[] { "\r\n" }, StringSplitOptions.None);
            

            foreach (var ai in a)
            {
                Console.WriteLine(ai);
                classes.Add(ai);
            }

            //这个地方也是读取文件，但是没用上
            FileStream myfs1 = new FileStream(colorsFile, FileMode.OpenOrCreate);
            StreamReader reader1 = new StreamReader(myfs1);
            string con1;
            con1 = reader1.ReadToEnd();
            var a1 = con1.Split(new string[] { "\r\n" }, StringSplitOptions.None);


            //因为我这里单独拿出来了
            colors.Add(new Scalar(0, 255,0));
            colors.Add(new Scalar(0, 0, 255));
            colors.Add(new Scalar(255, 0, 0));


            OpenCvSharp.Dnn.Net net = OpenCvSharp.Dnn.CvDnn.ReadNetFromTensorflow(modelWeights, textGraph);

            net.SetPreferableBackend(OpenCvSharp.Dnn.Net.Backend.OPENCV);    //opencv是使用intel编译的
            net.SetPreferableTarget(OpenCvSharp.Dnn.Net.Target.CPU);

            int ImgWidth = 224;
            int ImgHight = 224;

            Mat frame = Cv2.ImRead("1.jpg");
           
            Mat m_DstMat = frame.Clone();
            Size s2f = new Size(ImgWidth, ImgHight);
            Cv2.Resize(frame, frame, s2f);

            m_fWidthScale = (float)(m_DstMat.Cols * 1.0 / frame.Cols);
            m_fHeighScale = (float)(m_DstMat.Rows * 1.0 / frame.Rows);

            // Stop the program if reached end of video
            if (frame.Empty())
            {
                return;
            }


            blob = OpenCvSharp.Dnn.CvDnn.BlobFromImage(frame, 1.0, new Size(frame.Cols, frame.Rows), new Scalar(), true, false);

            net.SetInput(blob);

            List<string> outNames = new List<string>(2);
            outNames.Add("detection_out_final");  //此处是预测的边框点
            outNames.Add("detection_masks");  //这些名字设置和模型中的设置有关系，主要也是利用名称取索引相应的结果，此处是预测的掩模图

            List<Mat> outs = new List<Mat>();


            Mat outDetections;
            Mat outMasks ;

            //预测的图我选择了分开写，调用其他两种出不来，原因还在找，知道的可以交流交流，谢谢。详情可以打开net.ForWard()看一看
            //结果有三种：
            /*public Mat Forward([NullableAttribute(2)] string? outputName = null);
public void Forward(IEnumerable<Mat> outputBlobs, [NullableAttribute(2)] string? outputName = null);
public void Forward(IEnumerable<Mat> outputBlobs, IEnumerable<string> outBlobNames);*/
            outMasks = net.Forward("detection_masks");  
            outDetections = net.Forward("detection_out_final");


            int numDetections = outDetections.Size(2);
            int numClasses = outMasks.Size(1);


            // 预测框结果outDetections   掩膜结果outMasks 
            outDetections = outDetections.Reshape(1, (int)outDetections.Total() / 7);
            for (int i = 0; i < numDetections; ++i)
            {
                float score = outDetections.At<float>(i, 2);
                if (score > confThreshold)
                //if (score > 0.2)
                {
                    // Extract the bounding box
                    int classId = (int)(outDetections.At<float>(i, 1));
                    int left = (int)(frame.Cols * outDetections.At<float>(i, 3));
                    int top = (int)(frame.Rows * outDetections.At<float>(i, 4));
                    int right = (int)(frame.Cols * outDetections.At<float>(i, 5));
                    int bottom = (int)(frame.Rows * outDetections.At<float>(i, 6));

                    left = Math.Max(0, Math.Min(left, frame.Cols - 1));
                    top = Math.Max(0, Math.Min(top, frame.Rows - 1));
                    right = Math.Max(0, Math.Min(right, frame.Cols - 1));
                    bottom = Math.Max(0, Math.Min(bottom, frame.Rows - 1));
                    Rect box = new Rect(left, top, right - left + 1, bottom - top + 1);

                    /************************************************/
                    box.X = (int)Math.Round(box.X * m_fWidthScale);
                    box.Y = (int)Math.Round(box.Y * m_fHeighScale);
                    box.Width = (int)Math.Round(box.Width * m_fWidthScale);
                    box.Height = (int)Math.Round(box.Height * m_fHeighScale);
                    /************************************************/

                    // Extract the mask for the object
                    Mat objectMask = new Mat(outMasks.Size(2), outMasks.Size(3), MatType.CV_32F, outMasks.Ptr(i, classId));
                    // Draw bounding box, colorize and show the mask on the image

                    OpenCvSharp.Scalar s3 = new Scalar(255, 178, 50);

                    //Draw a rectangle displaying the bounding box
                    Cv2.Rectangle(m_DstMat, new Point(box.X, box.Y), new Point(box.X + box.Width, box.Y + box.Height), s3, 3);



                    //Get the label for the class name and its confidence
                    string label1;
                    label1 = score.ToString("F4");
                    string labelasdfa;
                    labelasdfa= classes[classId] + ":" + label1;



                    //Display the label at the top of the bounding box
                    int baseLine;
                    Size labelSize = Cv2.GetTextSize(labelasdfa, 0, 0.5, 1, out baseLine);
                    box.Y = Math.Max(box.Y, labelSize.Height);

                    Cv2.Rectangle(m_DstMat, new Point(box.X, box.Y - Math.Round(1.5 * labelSize.Height)), new Point(box.X + Math.Round(1.5 * labelSize.Width), box.Y + baseLine), new Scalar(255, 255, 255), Cv2.FILLED);

                    Cv2.PutText(m_DstMat, labelasdfa, new Point(box.X, box.Y), 0, 0.75, new Scalar(0, 0, 0), 1);


                    Scalar color = colors[classId % colors.Capacity];  

                    // Resize the mask, threshold, color and apply it on the image
                    Cv2.Resize(objectMask, objectMask, new Size(box.Width, box.Height));


                    Mat mask = objectMask;

                    Mat pos1 = new Mat(m_DstMat, box);
 

                    //Mat coloredRoi = (0.3 * color + 0.7 * m_DstMat(box));


                    Mat coloredRoi = new Mat(m_DstMat, box);
                    coloredRoi.ConvertTo(coloredRoi, MatType.CV_8UC3);



                    mask.ConvertTo(mask, MatType.CV_8U);
                    Cv2.FindContours(mask, out Point[][] contours,out HierarchyIndex[] hierarchy, RetrievalModes.CComp, ContourApproximationModes.ApproxSimple);

                    Cv2.DrawContours(coloredRoi, contours, -1, color, 5, LineTypes.Link8, hierarchy, 100);

                    Mat pos = new Mat(m_DstMat, box);
                    coloredRoi.CopyTo(pos, mask);


                }
            }


            double[] layersTimes = new double[] { };
            double freq = Cv2.GetTickFrequency() / 1000;
            double t = net.GetPerfProfile(out layersTimes) / freq;
            string label = "Inference time for a frame "+t.ToString("F4")+"ms";

            OpenCvSharp.Scalar s2 = new Scalar(0, 0, 0);
            Point xy = new Point(0, 15);
            Cv2.PutText(m_DstMat, label, xy, 0, 0.5, s2);

            Cv2.ImShow("Result", m_DstMat);
            Cv2.WaitKey(0);


        }



    }
}

至此结束！本来想分分让其更好看一些，太忙了，小伙伴们有空可以封装一下。欢迎互相交流。附送一张图(●'◡'●)！！！