C# OnnxRuntime部署LivePortrait实现快速、高质量的人像驱动视频生成

目录

效果

说明

项目

模型信息

代码

下载

---

效果

LivePortrait实现快速、高质量的人像驱动视频生成

说明

官网地址：https://github.com/KwaiVGI/LivePortrait

代码实现参考：https://github.com/hpc203/liveportrait-onnxrun

模型下载：onnx文件在百度云盘，链接: https://pan.baidu.com/s/13wjBFRHIIyCyBsgnBOKqsw 提取码: si95

项目

模型信息

appearance_feature_extractor.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：img
tensor：Float[1, 3, 256, 256]
---------------------------------------------------------------

Outputs
-------------------------
name：output
tensor：Float[1, 32, 16, 64, 64]
---------------------------------------------------------------

face_2dpose_106_static.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：data
tensor：Float[1, 3, 192, 192]
---------------------------------------------------------------

Outputs
-------------------------
name：fc1
tensor：Float[1, 212]
---------------------------------------------------------------

landmark.onnx
Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：input
tensor：Float[1, 3, 224, 224]
---------------------------------------------------------------

Outputs
-------------------------
name：output
tensor：Float[1, 214]
name：853
tensor：Float[1, 262]
name：856
tensor：Float[1, 406]
---------------------------------------------------------------

motion_extractor.onnx
Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：img
tensor：Float[1, 3, 256, 256]
---------------------------------------------------------------

Outputs
-------------------------
name：pitch
tensor：Float[1, 66]
name：yaw
tensor：Float[1, 66]
name：roll
tensor：Float[1, 66]
name：t
tensor：Float[1, 3]
name：exp
tensor：Float[1, 63]
name：scale
tensor：Float[1, 1]
name：kp
tensor：Float[1, 63]
---------------------------------------------------------------

retinaface_det_static.onnx
Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：input.1
tensor：Float[1, 3, 512, 512]
---------------------------------------------------------------

Outputs
-------------------------
name：448
tensor：Float[8192, 1]
name：471
tensor：Float[2048, 1]
name：494
tensor：Float[512, 1]
name：451
tensor：Float[8192, 4]
name：474
tensor：Float[2048, 4]
name：497
tensor：Float[512, 4]
name：454
tensor：Float[8192, 10]
name：477
tensor：Float[2048, 10]
name：500
tensor：Float[512, 10]
---------------------------------------------------------------

stitching.onnx

Model Properties
-------------------------
---------------------------------------------------------------

Inputs
-------------------------
name：input
tensor：Float[1, 126]
---------------------------------------------------------------

Outputs
-------------------------
name：output
tensor：Float[1, 65]
---------------------------------------------------------------

warping_spade.onnx

Inputs
feature_3d
name: feature_3d
tensor: float32[1,32,16,64,64]
kp_driving
name: kp_driving
tensor: float32[1,21,3]
kp_source
name: kp_source
tensor: float32[1,21,3]
Outputs
name: out
tensor: float32[1,3,512,512]

代码

using LivePortraitSharp;
using OpenCvSharp;
using System;
using System.Drawing;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace FaceFusionSharp
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        string fileFilter = "图片|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string startupPath = "";
        string source_path = "";
        string video_path = "";
        string videoFilter = "视频|*.mp4;*.avi;";
        LivePortraitPipeline livePortraitPipeline;

        /// <summary>
        /// 选择静态图像
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button2_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            ofd.InitialDirectory = Application.StartupPath + "\\test";
            if (ofd.ShowDialog() != DialogResult.OK) return;

            pictureBox1.Image = null;

            source_path = ofd.FileName;
            pictureBox1.Image = new Bitmap(source_path);
        }

        /// <summary>
        /// 驱动视频
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button3_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = videoFilter;
            ofd.InitialDirectory = Application.StartupPath + "\\test\\driving";
            if (ofd.ShowDialog() != DialogResult.OK) return;

            video_path = ofd.FileName;
            textBox1.Text = video_path;

            //读取第一帧显示
            VideoCapture vcapture = new VideoCapture(video_path);
            if (!vcapture.IsOpened())
            {
                MessageBox.Show("打开视频文件失败");
                video_path = "";
                return;
            }
            Mat frame = new Mat();
            if (vcapture.Read(frame))
            {
                pictureBox2.Image = new Bitmap(frame.ToMemoryStream());
                frame.Dispose();
            }
            else
            {
                MessageBox.Show("读取视频文件失败");
                video_path = "";
            }

        }

        private void button1_Click(object sender, EventArgs e)
        {
            if (source_path == "")
            {
                MessageBox.Show("请选择静态图");
                return;
            }

            if (video_path == "")
            {
                MessageBox.Show("请选择驱动视频");
                return;
            }

            bool saveDetVideo = false;
            if (checkBox1.Checked)
            {
                saveDetVideo = true;
            }
            else
            {
                saveDetVideo = false;
            }

            button1.Enabled = false;
            textBox1.Text = "";
            Application.DoEvents();

            Task.Factory.StartNew(() =>
            {

                string msg;
                if (!livePortraitPipeline.execute(source_path, video_path, saveDetVideo, out msg))
                {
                    this.Invoke(new Action(() =>
                    {
                        button1.Enabled = true;
                        textBox1.Text = msg;
                    }));
                }
                else
                {
                    this.Invoke(new Action(() =>
                    {
                        button1.Enabled = true;
                        textBox1.Text = "执行完成！";
                    }));
                }

            }, TaskCreationOptions.LongRunning);
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            livePortraitPipeline = new LivePortraitPipeline();

            source_path = "test\\0.jpg";
            pictureBox1.Image = new Bitmap(source_path);

            video_path = "test\\driving\\d0.mp4";
            //读取第一帧显示
            VideoCapture vcapture = new VideoCapture(video_path);
            if (!vcapture.IsOpened())
            {
                video_path = "";
                return;
            }
            Mat frame = new Mat();
            if (vcapture.Read(frame))
            {
                pictureBox2.Image = new Bitmap(frame.ToMemoryStream());
                frame.Dispose();
            }
            else
            {
                video_path = "";
            }
        }

    }
}

using LivePortraitSharp;
using OpenCvSharp;
using System;
using System.Drawing;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace FaceFusionSharp
{
    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        string fileFilter = "图片|*.bmp;*.jpg;*.jpeg;*.tiff;*.tiff;*.png";
        string startupPath = "";
        string source_path = "";
        string video_path = "";
        string videoFilter = "视频|*.mp4;*.avi;";
        LivePortraitPipeline livePortraitPipeline;

        /// <summary>
        /// 选择静态图像
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button2_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = fileFilter;
            ofd.InitialDirectory = Application.StartupPath + "\\test";
            if (ofd.ShowDialog() != DialogResult.OK) return;

            pictureBox1.Image = null;

            source_path = ofd.FileName;
            pictureBox1.Image = new Bitmap(source_path);
        }

        /// <summary>
        /// 驱动视频
        /// </summary>
        /// <param name="sender"></param>
        /// <param name="e"></param>
        private void button3_Click(object sender, EventArgs e)
        {
            OpenFileDialog ofd = new OpenFileDialog();
            ofd.Filter = videoFilter;
            ofd.InitialDirectory = Application.StartupPath + "\\test\\driving";
            if (ofd.ShowDialog() != DialogResult.OK) return;

            video_path = ofd.FileName;
            textBox1.Text = video_path;

            //读取第一帧显示
            VideoCapture vcapture = new VideoCapture(video_path);
            if (!vcapture.IsOpened())
            {
                MessageBox.Show("打开视频文件失败");
                video_path = "";
                return;
            }
            Mat frame = new Mat();
            if (vcapture.Read(frame))
            {
                pictureBox2.Image = new Bitmap(frame.ToMemoryStream());
                frame.Dispose();
            }
            else
            {
                MessageBox.Show("读取视频文件失败");
                video_path = "";
            }

        }

        private void button1_Click(object sender, EventArgs e)
        {
            if (source_path == "")
            {
                MessageBox.Show("请选择静态图");
                return;
            }

            if (video_path == "")
            {
                MessageBox.Show("请选择驱动视频");
                return;
            }

            bool saveDetVideo = false;
            if (checkBox1.Checked)
            {
                saveDetVideo = true;
            }
            else
            {
                saveDetVideo = false;
            }

            button1.Enabled = false;
            textBox1.Text = "";
            Application.DoEvents();

            Task.Factory.StartNew(() =>
            {

                string msg;
                if (!livePortraitPipeline.execute(source_path, video_path, saveDetVideo, out msg))
                {
                    this.Invoke(new Action(() =>
                    {
                        button1.Enabled = true;
                        textBox1.Text = msg;
                    }));
                }
                else
                {
                    this.Invoke(new Action(() =>
                    {
                        button1.Enabled = true;
                        textBox1.Text = "执行完成！";
                    }));
                }

            }, TaskCreationOptions.LongRunning);
        }

        private void Form1_Load(object sender, EventArgs e)
        {
            livePortraitPipeline = new LivePortraitPipeline();

            source_path = "test\\0.jpg";
            pictureBox1.Image = new Bitmap(source_path);

            video_path = "test\\driving\\d0.mp4";
            //读取第一帧显示
            VideoCapture vcapture = new VideoCapture(video_path);
            if (!vcapture.IsOpened())
            {
                video_path = "";
                return;
            }
            Mat frame = new Mat();
            if (vcapture.Read(frame))
            {
                pictureBox2.Image = new Bitmap(frame.ToMemoryStream());
                frame.Dispose();
            }
            else
            {
                video_path = "";
            }
        }

    }
}

下载

源码下载