【GStreamer】基于NTP+SEI的视频流传输时延测量

【GStreamer】基于NTP+SEI的视频流传输时延测量

本文以H.264视频流为例，用GStreamer实现插入和提取SEI(Supplemental Enhancement Information)，实现视频流传输时延的测量。

原理¹

用GStreamer实现的方案

sender

第一关键节点是funnel，翻译过来就是漏斗，以GstBuffer为最小粒度对两路数据流做merge，形象点讲就是appsrc产生绿豆，x264enc产生红豆，绿豆红豆一字纵队过漏斗，每颗豆豆代表GstBuffer，是以alignment=au²为最小粒度的NAL。

第二关键节点是appsrc，通过need-data signal³向pipeline中插入SEI

gst-launch语法的pipeline如下：

gst-launch-1.0 funnel name=f \
appsrc name=appsrc-h264-sei do-timestamp=true block=true is-live=true ! video/x-h264, stream-format=byte-stream, alignment=au ! queue ! f. \
videotestsrc is-live=true ! x264enc ! video/x-h264, stream-format=byte-stream, alignment=au, profile=baseline ! queue ! f. \
f. ! queue ! h264parse ! video/x-h264, stream-format=byte-stream, alignment=au ! rtph264pay ! udpsink sync=false clients=127.0.0.1:5004

注册到need-data signal的need_data_callback实现：

static void need_data_callback(GstElement *appsrc, guint unused,
                               gpointer udata) {
  GST_LOG("need_data_callback");
  GstBuffer *buffer;
  GstFlowReturn ret;
  static uint64_t next_ms_time_insert_sei = 0;
  struct timespec one_ms;
  struct timespec rem;
  uint8_t *h264_sei = NULL;
  size_t length = 0;

  one_ms.tv_sec = 0;
  one_ms.tv_nsec = 1000000;

  while (now_ms() <= next_ms_time_insert_sei) {
    GST_TRACE("sleep to wait time trigger");
    nanosleep(&one_ms, &rem);
  }

  if (!h264_sei_ntp_new(&h264_sei, &length)) {
    GST_ERROR("h264_sei_ntp_new failed");
    return;
  }

  if (NULL != h264_sei && length > 0) {
    buffer =
        gst_buffer_new_allocate(NULL, START_CODE_PREFIX_BYTES + length, NULL);
    if (NULL != buffer) {
      // fill start_code_prefix: 0x00000001
      uint8_t start_code_prefix[] = START_CODE_PREFIX;
      gst_buffer_fill(buffer, 0, start_code_prefix, START_CODE_PREFIX_BYTES);
      // fill H.264 SEI
      size_t bytes_copied =
          gst_buffer_fill(buffer, START_CODE_PREFIX_BYTES, h264_sei, length);
      if (bytes_copied == length) {
        g_signal_emit_by_name(appsrc, "push-buffer", buffer, &ret);
        GST_DEBUG("H264 SEI NTP timestamp inserted");
      } else {
        GST_ERROR("GstBuffer.fill without all bytes copied");
      }
    } else {
      GST_ERROR("GstBuffer.new_allocate failed");
    }
    gst_buffer_unref(buffer);
  }

  next_ms_time_insert_sei = now_ms() + 1000;
  free(h264_sei);
}

receiver

关键节点是identity，对于每个GstBuffer的到来都会触发handoff signal，接下来的识别处理工作就交给handoff_callback处理

gst-launch语法的pipeline如下：

gst-launch-1.0 udpsrc uri=udp://127.0.0.1:5004 caps="application/x-rtp, media=video, encoding-name=H264" ! rtph264depay ! video/x-h264, stream-format=byte-stream, alignment=nal ! identity name=identity ! fakesink

注意到alignment=nal跟sender不一样，因为rtph264depay输出为alignment=au时会把SEI丢弃掉⁴

注册到handoff signal的handoff_callback实现：

static void handoff_callback(GstElement *identity, GstBuffer *buffer,
                             gpointer user_data) {
  GST_TRACE("handoff_callback");
  GstMapInfo info = GST_MAP_INFO_INIT;
  GstH264NalParser *nalparser = NULL;
  GstH264NalUnit nalu;

  if (gst_buffer_map(buffer, &info, GST_MAP_READ)) {
    nalparser = gst_h264_nal_parser_new();
    if (NULL != nalparser) {
      if (GST_H264_PARSER_OK ==
          gst_h264_parser_identify_nalu_unchecked(nalparser, info.data, 0,
                                                  info.size, &nalu)) {
        // if (info.size < 100) GST_LOG("buffer info size %ld", info.size);
        if (GST_H264_NAL_SEI == nalu.type) {
          GST_LOG(
              "identify sei nalu with size = %d, offset = %d, sc_offset = %d",
              nalu.size, nalu.offset, nalu.sc_offset);
          int64_t delay = -1;
          if (TRUE ==
              h264_sei_ntp_parse(nalu.data + nalu.offset, nalu.size, &delay)) {
            GST_LOG("delay = %ld ms", delay);
          }
        }
      } else {
        GST_WARNING("gst_h264_parser_identify_nalu_unchecked failed");
      }

      gst_h264_nal_parser_free(nalparser);
    } else {
      GST_WARNING("gst_h264_nal_parser_new failed");
    }

    gst_buffer_unmap(buffer, &info);
  } else {
    GST_WARNING("gst_buffer_map failed");
  }
}

NTP SEI generator and parser

基于aizvorski/h264bitstream实现
关于H.264 SEI的数据结构参考：FFmpeg从入门到精通——进阶篇，SEI那些事儿

// h264_sei_ntp.h
#ifndef H264_SEI_NTP
#define H264_SEI_NTP

#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

// #define H264_SEI_UUID_NTP_TIMESTAMP {0x60, 0x2b, 0x0d, 0xb6, 0x2d, 0x3d,
// 0x44, 0xb5, 0xab, 0x9e, 0xec, 0x8a, 0xd7, 0x1f, 0x3f, 0x8e} test
// emulation_prevention_three_byte
#define H264_SEI_UUID_NTP_TIMESTAMP                                            \
  {                                                                            \
    0x00, 0x00, 0x01, 0x00, 0x00, 0x02, 0x00, 0x00, 0x03, 0x9e, 0xec, 0x8a,    \
        0xd7, 0x1f, 0x3f, 0x8e                                                 \
  }
#define START_CODE_PREFIX_BYTES 4
#define START_CODE_PREFIX                                                      \
  { 0x00, 0x00, 0x00, 0x01 }

uint64_t now_ms();

bool h264_sei_ntp_new(uint8_t **h264_sei, size_t *length);

bool h264_sei_ntp_parse(uint8_t *h264_sei, size_t length, int64_t *delay);

#endif

// h264_sei_ntp.c
#include "h264_sei_ntp.h"
#include "h264_stream.h" // https://github.com/D-Y-Innovations/h264bitstream/blob/master/h264_stream.h
#include <math.h>
#include <stdlib.h>
#include <time.h>

#define NALU_BUFFER_MAX_SIZE 50
#define SEI_PAYLOAD_SIZE 24
#define H264_SEI_NTP_UUID_SIZE 16

/*
start_code_prefix_one_3bytes: 0x000001
nal_unit_type: 0x06, Supplemental enhancement information (SEI)
SEI payload_type_byte: 0x05,
SEI payload_size_byte: 0x18
uuid_iso_iec_11578: 602b0db6-2d3d-44b5-ab9e-ec8ad71f3f8e
user_data_payload_byte: 0x0000012345678912
rbsp_trailing_bits(): 0x80

emulation_prevention_three_byte: 0x03
*/

uint64_t now_ms() {
  long ms;  // Milliseconds
  time_t s; // Seconds
  struct timespec spec;

  clock_gettime(CLOCK_REALTIME, &spec);

  s = spec.tv_sec;
  ms = round(spec.tv_nsec / 1.0e6); // Convert nanoseconds to milliseconds
  if (ms > 999) {
    s++;
    ms = 0;
  }

  return s * 1000 + ms;
}

/**
 * @param h264_sei H264 SEI buffer with start_code_prefix_one_3bytes,
 * transfer-full, handler should free memory after used
 * @param length   h264_sei 的长度
 * @return         如果执行成功则返回true，否则返回false
 * @since          1.0
 */
bool h264_sei_ntp_new(uint8_t **h264_sei, size_t *length) {
  uint8_t sei_uuid[] = H264_SEI_UUID_NTP_TIMESTAMP;
  uint8_t buffer[NALU_BUFFER_MAX_SIZE] = {0};
  uint8_t payloadData[SEI_PAYLOAD_SIZE] = {0};

  h264_stream_t *h264_nalu_sei = h264_new();
  h264_nalu_sei->nal->nal_ref_idc = NAL_REF_IDC_PRIORITY_DISPOSABLE;
  h264_nalu_sei->nal->nal_unit_type = NAL_UNIT_TYPE_SEI;

  memcpy(payloadData, sei_uuid, sizeof(sei_uuid));
  uint64_t milli_time = now_ms();
  memcpy(&(payloadData[16]), &milli_time, sizeof(uint64_t)); // little-endian

  sei_t *seis[1] = {sei_new()};
  sei_t *sei = seis[0];
  sei->payloadType = SEI_TYPE_USER_DATA_UNREGISTERED;
  sei->payloadSize = SEI_PAYLOAD_SIZE;
  sei->data = payloadData;

  h264_nalu_sei->seis = seis;
  h264_nalu_sei->num_seis = 1;

  int len = write_nal_unit(h264_nalu_sei, buffer, NALU_BUFFER_MAX_SIZE);
  if (len <= 0) {
    return false;
  }

  uint8_t *h264_sei_tmp = malloc(len);
  if (NULL == h264_sei_tmp)
    return false;
  memcpy(h264_sei_tmp, buffer, len);

  *length = len;
  *h264_sei = h264_sei_tmp;
  return true;
}

/**
 * @param h264_sei H264 SEI buffer without start_code_prefix_one_3bytes
 * @param length   h264_sei 的长度
 * @param delay    返回时延，单位ms
 * @return         如果执行成功则返回true，否则返回false
 * @since          1.0
 */
bool h264_sei_ntp_parse(uint8_t *h264_sei, size_t length, int64_t *delay) {
  uint8_t sei_uuid[] = H264_SEI_UUID_NTP_TIMESTAMP;

  h264_stream_t *h264_nalu_sei = h264_new();
  if (read_nal_unit(h264_nalu_sei, h264_sei, length) > 0) {
    if (1 == h264_nalu_sei->num_seis) {
      sei_t *sei = h264_nalu_sei->seis[0];
      if (SEI_TYPE_USER_DATA_UNREGISTERED == sei->payloadType) {
        if (SEI_PAYLOAD_SIZE == sei->payloadSize) {
          if (0 == memcmp(sei->data, sei_uuid, H264_SEI_NTP_UUID_SIZE)) {
            uint64_t timestamp = 0;
            memcpy(&timestamp, sei->data + H264_SEI_NTP_UUID_SIZE,
                   sizeof(timestamp));
            *delay = now_ms() - timestamp;
            if (*delay >= 0) {
              return true;
            } else {
              printf("delay <0\n");
            }
          } else {
            printf("memcmp != 0\n");
          }
        } else {
          printf("payloadSize !== SEI_PAYLOAD_SIZE\n");
        }
      } else {
        printf("payloadType !== SEI_TYPE_USER_DATA_UNREGISTERED\n");
      }
    } else {
      printf("num_seis !== 1\n");
    }
  } else {
    printf("read_nal_unit <= 0\n");
  }
  return false;
}

---

1. 播放器技术分享（5）：延时优化 ↩︎

2. What is the alignment capability in video/x-h264 ↩︎

3. appsrc-stream2.c ↩︎

4. gstrtph264depay.c ↩︎