gstreamer的I420转jpeg格式数据产生耗时

1、将I430数据转化为jpg数据

JPegEncoder.h

#pragma once
#include <cstdint>
#include <vector>
#include <functional>
#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include "Resolution.h"
#include "VideoSettings.h"

namespace sinftech {
namespace tv {
class JpegEncoder {
public:
  using Handler = std::function<void(const uint8_t* data, size_t size, const Resolution& resolution)>;
  JpegEncoder(Handler&& handler);
  ~JpegEncoder();
  void encode(const uint8_t* data, size_t size, const Resolution& resolution);
  void start(const VideoSettings& settings);
  void stop();

private:
  static GstFlowReturn _onEncoded(GstAppSink *appsink, gpointer user_data);
  GstElement* _pipeline;
  GstElement* _source;
  GstElement* _queue0;
  GstElement* _convert;
  GstElement* _scale;
  GstElement* _rate;
  GstElement* _capsfilter;
  GstElement* _queue1;
  GstElement* _encode;
  GstElement* _queue2;
  GstElement* _sink;
  Handler _handler;
};
}//namespace tv
}//namespace sinftech

JPegEncoder.cpp

#include "JpegEncoder.h"
#include <gst/app/gstappsrc.h>
#include <gst/video/video.h>
#include "Check.h"
#include <windows.h>

namespace sinftech {
namespace tv {

GstFlowReturn JpegEncoder::_onEncoded(GstAppSink *appsink, gpointer user_data) {
  JpegEncoder* pThis = static_cast<JpegEncoder*>(user_data);
  GstSample* sample = gst_app_sink_pull_sample(appsink);
  if (sample != NULL) {
    GstCaps* caps = gst_sample_get_caps(sample);
    gint width, height;
    GstStructure* structure = gst_caps_get_structure(caps, 0);
    gst_structure_get_int(structure, "width", &width);
    gst_structure_get_int(structure, "height", &height);
    GstBuffer* buffer = gst_sample_get_buffer(sample);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_READ)) {
        if (pThis->_handler) {
          pThis->_handler(map.data, map.size, {uint16_t(width), uint16_t(height)});
        }
        gst_buffer_unmap(buffer, &map);
      }
    }
    gst_sample_unref(sample);
  }
  return GST_FLOW_OK;
}

JpegEncoder::JpegEncoder(Handler&& handler)
  : _handler(std::move(handler)) {
  gst_init(nullptr, nullptr);
  _pipeline   = chk(gst_pipeline_new(nullptr));
  _source     = chk(gst_element_factory_make("appsrc",       nullptr));
  _queue0     = chk(gst_element_factory_make("queue",        nullptr));
  _convert    = chk(gst_element_factory_make("videoconvert", nullptr));
  _scale      = chk(gst_element_factory_make("videoscale",   nullptr));
  _rate       = chk(gst_element_factory_make("videorate",    nullptr));
  _capsfilter = chk(gst_element_factory_make("capsfilter",   nullptr));
  _queue1     = chk(gst_element_factory_make("queue",        nullptr));
  _encode     = chk(gst_element_factory_make("jpegenc",      nullptr));
  _queue2     = chk(gst_element_factory_make("queue",        nullptr));
  _sink       = chk(gst_element_factory_make("appsink",      nullptr));

  g_object_set(_source, "is-live", true, "do-timestamp", true, "format", GST_FORMAT_TIME, NULL);
  g_object_set(_queue0, "leaky", 2, NULL);
  g_object_set(_capsfilter, "caps-change-mode", 1, NULL);
  g_object_set(_queue1, "leaky", 2, NULL);
  g_object_set(_queue2, "leaky", 2, NULL);
  g_object_set(_sink, "sync", false, "enable-last-sample", false, "drop", true, "max-buffers", 200, "max-lateness", 1000000000ll, NULL);

  GstAppSinkCallbacks callbacks{nullptr, nullptr, &_onEncoded};
  gst_app_sink_set_callbacks(GST_APP_SINK(_sink), &callbacks, this, nullptr);

  gst_bin_add_many(GST_BIN(_pipeline), _source, _queue0, _convert, _scale, _rate, _capsfilter, _queue1, _encode, _queue2, _sink, NULL);
  gst_element_link_many(               _source, _queue0, _convert, _scale, _rate, _capsfilter, _queue1, _encode, _queue2, _sink, NULL);
}

JpegEncoder::~JpegEncoder() {
  gst_element_set_state(_pipeline, GST_STATE_NULL);
  gst_object_unref(_pipeline);
}

void JpegEncoder::start(const VideoSettings& settings) {
  stop();
  GstCaps* caps = gst_caps_new_simple(
    "video/x-raw",
    "format", G_TYPE_STRING, "I420",
    "width", G_TYPE_INT, settings.resolution.width,
    "height", G_TYPE_INT, settings.resolution.height,
    "framerate", GST_TYPE_FRACTION, settings.fps, 1,
    NULL);
  g_object_set(_capsfilter, "caps", caps, NULL);
  gst_caps_unref(caps);

  gst_element_set_state(_pipeline, GST_STATE_PLAYING);
}

void JpegEncoder::stop() {
  gst_element_set_state(_pipeline, GST_STATE_NULL);
}

void JpegEncoder::encode(const uint8_t* data, size_t size, const Resolution& resolution) {
  GstState state = GST_STATE_NULL;
  GstState pending = GST_STATE_NULL;
  gst_element_get_state(_pipeline, &state, &pending, 0);
  if (state == GST_STATE_PLAYING || pending == GST_STATE_PLAYING) {
    GstBuffer* buffer = gst_buffer_new_and_alloc(size);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_WRITE)) {
        memcpy(map.data, data, size);
        gst_buffer_unmap(buffer, &map);
        GstVideoInfo videoInfo;
        gst_video_info_set_format(&videoInfo, GST_VIDEO_FORMAT_I420, resolution.width, resolution.height);
        GstCaps* caps = gst_video_info_to_caps(&videoInfo);
        GstSample* sample = gst_sample_new(buffer, caps, nullptr, nullptr);
        gst_caps_unref(caps);
        if (sample != NULL) {
          gst_app_src_push_sample(GST_APP_SRC(_source), sample);
          gst_sample_unref(sample);
        }
      }
      gst_buffer_unref(buffer);
    }
  }
}

}//namespace tv
}//namespace sinftech

测试发现插件rate产生耗时(gstreamer在同步时间产生耗时), 并且一个输入会产生多个输出。耗时解释如下:

gstreamer的videorate插件的耗时取决于多个因素,包括输入视频帧率、输出视频帧率以及系统硬件性能等。在实际使用中,可以通过测试和性能优化来评估具体情况。

一般来说,videorate插件的耗时较低,因为它只是简单地调整视频帧的时间戳而不涉及复杂的图像处理操作。但是,如果输入和输出之间的帧率差异很大或者需要进行额外的转码操作,则可能会增加一些延迟。

现修改如下:

#include "JpegEncoder.h"
#include <gst/app/gstappsrc.h>
#include <gst/video/video.h>
#include "Check.h"
#include <windows.h>

#include <string>
#include <chrono>
#include <ctime>

namespace sinftech {
namespace tv {

GstFlowReturn JpegEncoder::_onEncoded(GstAppSink *appsink, gpointer user_data) {
  JpegEncoder* pThis = static_cast<JpegEncoder*>(user_data);
  GstSample* sample = gst_app_sink_pull_sample(appsink);
  if (sample != NULL) {
    GstCaps* caps = gst_sample_get_caps(sample);
    gint width, height;
    GstStructure* structure = gst_caps_get_structure(caps, 0);
    gst_structure_get_int(structure, "width", &width);
    gst_structure_get_int(structure, "height", &height);
    GstBuffer* buffer = gst_sample_get_buffer(sample);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_READ)) {
        if (pThis->_handler) {
          pThis->_handler(map.data, map.size, {uint16_t(width), uint16_t(height)});
          //pThis->SaveVideoDatToFile(map.data, map.size, 2);
        }
        gst_buffer_unmap(buffer, &map);
      }
    }
    gst_sample_unref(sample);
  }
  return GST_FLOW_OK;
}

JpegEncoder::JpegEncoder(Handler&& handler, bool bEncoderEnable)
  : _handler(std::move(handler)) {

    m_bEncoderEnable = bEncoderEnable;
  //gst_debug_set_active(TRUE);
  //gst_debug_set_default_threshold(GST_LEVEL_WARNING);
    if (m_bEncoderEnable)
    {
        gst_init(nullptr, nullptr);
        _pipeline = chk(gst_pipeline_new(nullptr));
        _source = chk(gst_element_factory_make("appsrc", nullptr));
        _queue0 = chk(gst_element_factory_make("queue", nullptr));
        _convert = chk(gst_element_factory_make("videoconvert", nullptr));
        _scale = chk(gst_element_factory_make("videoscale", nullptr));
        _capsfilter = chk(gst_element_factory_make("capsfilter", nullptr));
        _queue1 = chk(gst_element_factory_make("queue", nullptr));
        _encode = chk(gst_element_factory_make("jpegenc", nullptr));
        _queue2 = chk(gst_element_factory_make("queue", nullptr));
        _sink = chk(gst_element_factory_make("appsink", nullptr));

        g_object_set(_source, "is-live", true, "do-timestamp", true, "format", GST_FORMAT_TIME, NULL);
        g_object_set(_queue0, "leaky", 2, NULL);
        g_object_set(_capsfilter, "caps-change-mode", 1, NULL);
        g_object_set(_queue1, "leaky", 2, NULL);
        g_object_set(_queue2, "leaky", 2, NULL);
        //g_object_set(_sink, "sync", false, "enable-last-sample", false, "drop", true, "max-buffers", 200, "max-lateness", 1000000000ll, NULL);
        g_object_set(_sink, "sync", true, "enable-last-sample", true, "drop", true, "max-buffers", 1, "max-lateness", 1, NULL);

        GstAppSinkCallbacks callbacks{ nullptr, nullptr, &_onEncoded };
        gst_app_sink_set_callbacks(GST_APP_SINK(_sink), &callbacks, this, nullptr);

        gst_bin_add_many(GST_BIN(_pipeline), _source, _queue0, _convert, _scale, _capsfilter, _queue1, _encode, _queue2, _sink, NULL);
        gst_element_link_many(_source, _queue0, _convert, _scale, _capsfilter, _queue1, _encode, _queue2, _sink, NULL);
    }
}

JpegEncoder::~JpegEncoder() {
    if (m_bEncoderEnable)
    {
        gst_element_set_state(_pipeline, GST_STATE_NULL);
        gst_object_unref(_pipeline);
    }
}

void JpegEncoder::start(const VideoSettings& settings) 
{
    if (!m_bEncoderEnable)
        return;

	if (0 != settings.fps && settings != _settings)
	{
		_settings = settings;
		pause();
		GstCaps* caps = gst_caps_new_simple(
			"video/x-raw",
			"format", G_TYPE_STRING, "I420",
			"width", G_TYPE_INT, settings.resolution.width,
			"height", G_TYPE_INT, settings.resolution.height,
			"framerate", GST_TYPE_FRACTION, settings.fps, 1,
			NULL);
		g_object_set(_capsfilter, "caps", caps, NULL);
		gst_caps_unref(caps);

		gst_element_set_state(_pipeline, GST_STATE_PLAYING);
	}
	else
	{
		//do nothing
	}
}

void JpegEncoder::pause(void)
{
    if (!m_bEncoderEnable)
        return;

	gst_element_set_state(_pipeline, GST_STATE_NULL);
}

void JpegEncoder::stop() {
    if (!m_bEncoderEnable)
        return;

	pause();
	_settings.fps = 0;  //force reset setting
}

void JpegEncoder::encode(const uint8_t* data, size_t size, const Resolution& resolution) {

    if (!m_bEncoderEnable)
        return;

  //SaveVideoDatToFile(data, size, 1);

  GstState state = GST_STATE_NULL;
  GstState pending = GST_STATE_NULL;
  gst_element_get_state(_pipeline, &state, &pending, 0);
  if (state == GST_STATE_PLAYING || pending == GST_STATE_PLAYING) {
    GstBuffer* buffer = gst_buffer_new_and_alloc(size);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_WRITE)) {
        memcpy(map.data, data, size);
        gst_buffer_unmap(buffer, &map);
        GstVideoInfo videoInfo;
        gst_video_info_set_format(&videoInfo, GST_VIDEO_FORMAT_I420, resolution.width, resolution.height);
        GstCaps* caps = gst_video_info_to_caps(&videoInfo);
        GstSample* sample = gst_sample_new(buffer, caps, nullptr, nullptr);
        gst_caps_unref(caps);
        if (sample != NULL) {
          gst_app_src_push_sample(GST_APP_SRC(_source), sample);
          gst_sample_unref(sample);
        }
      }
      gst_buffer_unref(buffer);
    }
  }
}

std::string JpegEncoder::GetCurFormatTime()
{
    auto now = std::chrono::system_clock::now();
    std::time_t time = std::chrono::system_clock::to_time_t(now);
    std::tm* tm = std::localtime(&time);

    auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(
        now.time_since_epoch()).count() % 1000;

    return std::move(std::to_string(tm->tm_year + 1900)
        + "-" + std::to_string(tm->tm_mon + 1)
        + "-" + std::to_string(tm->tm_mday)
        + "_" + std::to_string(tm->tm_hour)
        + "-" + std::to_string(tm->tm_min)
        + "-" + std::to_string(tm->tm_sec)
        + "." + std::to_string(ms));
}

void JpegEncoder::SaveVideoDatToFile(const uint8_t* data, size_t size, int type)
{
    std::string fileName;
    if (type == 1)
    {
        fileName = "input_" + GetCurFormatTime() + ".yuv";
    }
    else if (type == 2)
    {
        fileName = "output_" + GetCurFormatTime() + ".jpg";
    }
    
    FILE* fp = fopen(fileName.c_str(), "wb+");
    fwrite(data, size, 1, fp);
    fclose(fp);
}

}//namespace tv
}//namespace sinftech

2、jpeg数据转化为I420

JPegDecoder.h

#pragma once
#include <cstdint>
#include <vector>
#include <thread>
#include <gst/gst.h>
#include <gst/app/gstappsink.h>
#include "Resolution.h"
#include "VideoSettings.h"

namespace sinftech {
namespace tv {
class JpegDecoder {
public:
  using Handler = std::function<void(const uint8_t* data, size_t size, const Resolution& resolution)>;
  JpegDecoder(Handler&& handler);
  ~JpegDecoder();
  void decode(const uint8_t* data, size_t size);

private:
  static GstFlowReturn _onDecoded(GstAppSink *appsink, gpointer user_data);
  GstElement* _pipeline;
  GstElement* _source;
  GstElement* _queue0;
  GstElement* _decode;
  GstElement* _queue1;
  GstElement* _sink;
  Handler _handler;
};
}//namespace tv
}//namespace sinftech

JPegDecoder.cpp

#include "JpegDecoder.h"
#include <gst/app/gstappsrc.h>
#include <gst/video/video.h>
#include "Check.h"
#include <util/Log.h>

namespace sinftech {
namespace tv {

GstFlowReturn JpegDecoder::_onDecoded(GstAppSink *appsink, gpointer user_data) {
  JpegDecoder* pThis = static_cast<JpegDecoder*>(user_data);
  GstSample* sample = gst_app_sink_pull_sample(appsink);
  if (sample != NULL) {
    GstCaps* caps = gst_sample_get_caps(sample);
    gint width, height;
    GstStructure* structure = gst_caps_get_structure(caps, 0);
    gst_structure_get_int(structure, "width", &width);
    gst_structure_get_int(structure, "height", &height);
    GstBuffer* buffer = gst_sample_get_buffer(sample);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_READ)) {
        if (pThis->_handler) {
          try {
            pThis->_handler(map.data, map.size, {uint16_t(width), uint16_t(height)});
          } catch (const std::exception& e) {
            util::log(std::string("jpeg decoder exception: ") + e.what());
            throw;
          } catch (...) {
            util::log("jpeg decoder exception: unknown");
            throw;
          }
        }
        gst_buffer_unmap(buffer, &map);
      }
    }
    gst_sample_unref(sample);
  }
  return GST_FLOW_OK;
}

JpegDecoder::JpegDecoder(Handler&& handler)
  : _handler(std::move(handler)) {
  gst_init(nullptr, nullptr);
  _pipeline = chk(gst_pipeline_new(nullptr));
  _source   = chk(gst_element_factory_make("appsrc",  nullptr));
  _queue0   = chk(gst_element_factory_make("queue",   nullptr));
  _decode   = chk(gst_element_factory_make("jpegdec", nullptr));
  _queue1   = chk(gst_element_factory_make("queue",   nullptr));
  _sink     = chk(gst_element_factory_make("appsink", nullptr));

  g_object_set(_source, "is-live", true, "do-timestamp", true, "format", GST_FORMAT_TIME, NULL);
  g_object_set(_queue0, "leaky", 2, NULL);
  g_object_set(_queue1, "leaky", 2, NULL);
  GstCaps* caps = gst_caps_new_simple("video/x-raw", "format", G_TYPE_STRING, "I420", NULL);
  g_object_set(_sink, "caps", caps, "sync", false, "enable-last-sample", false, "drop", true, "max-buffers", 200, "max-lateness", 1000000000ll, NULL);
  gst_caps_unref(caps);

  GstAppSinkCallbacks callbacks{nullptr, nullptr, &_onDecoded};
  gst_app_sink_set_callbacks(GST_APP_SINK(_sink), &callbacks, this, nullptr);

  gst_bin_add_many(GST_BIN(_pipeline), _source, _queue0, _decode, _queue1, _sink, NULL);
  gst_element_link_many(               _source, _queue0, _decode, _queue1, _sink, NULL);

  gst_element_set_state(_pipeline, GST_STATE_PLAYING);
}

JpegDecoder::~JpegDecoder() {
  gst_element_set_state(_pipeline, GST_STATE_NULL);
  gst_object_unref(_pipeline);
}

void JpegDecoder::decode(const uint8_t* data, size_t size) {
  GstState state = GST_STATE_NULL;
  GstState pending = GST_STATE_NULL;
  gst_element_get_state(_pipeline, &state, &pending, 0);
  if (state == GST_STATE_PLAYING || pending == GST_STATE_PLAYING) {
    GstBuffer* buffer = gst_buffer_new_and_alloc(size);
    if (buffer != NULL) {
      GstMapInfo map;
      if (gst_buffer_map(buffer, &map, GST_MAP_WRITE)) {
        memcpy(map.data, data, size);
        gst_buffer_unmap(buffer, &map);
        GstCaps* caps = gst_caps_new_simple("image/jpeg", NULL);
        GstSample* sample = gst_sample_new(buffer, caps, nullptr, nullptr);
        gst_caps_unref(caps);
        if (sample != NULL) {
          gst_app_src_push_sample(GST_APP_SRC(_source), sample);
          gst_sample_unref(sample);
        }
      }
      gst_buffer_unref(buffer);
    }
  }
}

}//namespace tv
}//namespace sinftech

  • 0
    点赞
  • 2
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值