1)ffmpeg如何判断一帧数据是正确的?ffmpeg有没有错误处理的模式,能使花屏的帧(h264格式的)不显示出来?
2) H264网络传输过程中丢包造成马赛克问题?
原因:
1. 接收网络数据包后没有调整包的顺序,譬如说接受包的顺序是1,3,4,2,如果没有调整顺序的话,发送给解码器的顺序也是1,3,4,2,这样肯定会出现马赛克 ;
2. 接收网络数据包后没有没有合并数据包,众所周知,一个Video帧可能被分割成多个网络数据包传送,譬如说是1,2,3,如果在接受端没有将这三个包合并成一个Video帧发送给解码器,而是当成三个Video帧发送给解码器,也肯定会出现马赛克 ;
3. 没有正确处理好网络丢包的情况,如果所丢的数据包正好传送的是另外一个帧所参考的的数据,这样也会出现马赛克 ;
4. 解码器有问题,如果播放本地文件也出现马赛克的话。
解决方法:
1.服务器软件用多线程:
(1)主线程:读出(看你的图象具体怎么上PC机了)一帧视频数据,送给拆分线程。
(2)拆分线程:接到一帧视频,开始拆包、做帧标记、打序列号,送给发送线程。
(3)发送线程:用RTP socket把封装好的数据包发给客户端。此socket是点对多点、单向 有根方式的组播套接字,实际上是基于UDP派生的,但他用到了RTP和RTCP(实时传输 协议和实时传输控制协议),如果你传输的不是实时数据,直接用UDP就行了。
2.客户端软件结构一般用多线程,线程间用事件对象进行同步,而共享数据区用临界区对象进
行同步。
(1)主线程:接收网络数据包的线程,优先级最高,可以尽量保证不丢数据,也采用RTP协 议,用网络事件来触发。主线程接收到视频数据包后,将数据放入一个链表中,然后用事件对象触发组装线程。
(2)组装线程:从链表中读出数据包,要进行帧确认、排序等工作,当把一帧图象的所有 包都取到时,再调用组装模块(可以是一个函数),将这些数据包组装成完整的一个帧,然后送到解压线程。
(3)若干解压播放线程。主要考虑到如果你客户端软件想同时播放多画面,比如说4画面图 象,就要用4个解压播放线程。
(4)至于图象存储,要看你的客户需要怎么存了,如果是手工存当然不需要单开线程,如果 是规定定时存或在某个事件发生时自动存盘,就需要单开一个线程,由定时器到时消息或此事件发生来触发。
后来我们项目也将图象送上了广域网和Internet,主要就是传输的速率明显慢了。把服务器软件放在Web服务器上,用UDP点对点的方式给提出视频服务的Internet客户端发出相应视频帧。还可以实现对每个客户端的各种服务请求的响应和控制。
建议:
(1)Winsock传输部分最好不要用MFC提供的类,自己用API做,但是应用 程序可以用MFC做,
(2)在局域网上用点对多点的组播方式传输时,基于RTP和RTCP协议来做。 服务器方得拆包传,每包最好不要大于2K,以我去年做的项目经验, 用1.5K比较好,每包间隔1-2毫秒。发送方给每包打上时间戳和序列 号,接收方重新排序和组装。
(3)如果是点对点传,应该基于UDP协议,可以一帧一帧的传,我最大传过 30K。没问题,我已经在实际项目中试过,JPEG格式的视频流在局域网 中1秒15帧,一帧12K;在广域网中1秒中4帧,一帧15K。
(4) 如果你传输的是监控的告警数据,要求准确性,你必须基于TCP协议,用点对点方式传输。
VLC 对应的解决策略:
static inline uint16_t rtp_seq (const block_t *block)
{
assert (block->i_buffer >= 4);
return GetWBE (block->p_buffer + 2);
}
#define GetWBE( p ) U16_AT( p )
/* MSB (big endian)/LSB (little endian) conversions - network order is always
* MSB, and should be used for both network communications and files. */
LIBVLC_USED
static inline uint16_t U16_AT( const void * _p )
{
const uint8_t * p = (const uint8_t *)_p;
return ( ((uint16_t)p[0] << 8) | p[1] );
}
/** State for a RTP session: */
struct rtp_session_t
{
rtp_source_t **srcv;
unsigned srcc;
uint8_t ptc;
rtp_pt_t *ptv;
};
/** State for an RTP source */
struct rtp_source_t
{
uint32_t ssrc;
uint32_t jitter; /* interarrival delay jitter estimate */
mtime_t last_rx; /* last received packet local timestamp */
uint32_t last_ts; /* last received packet RTP timestamp */
uint32_t ref_rtp; /* sender RTP timestamp reference */
mtime_t ref_ntp; /* sender NTP timestamp reference */
uint16_t bad_seq; /* tentatively next expected sequence for resync */
uint16_t max_seq; /* next expected sequence */
uint16_t last_seq; /* sequence of the next dequeued packet */
block_t *blocks; /* re-ordered blocks queue */
void *opaque[0]; /* Per-source private payload data */
};
/**
* Destroys an RTP source and its associated streams.
*/
static void
rtp_source_destroy (demux_t *demux, const rtp_session_t *session,
rtp_source_t *source)
{
msg_Dbg (demux, "removing RTP source (%08x)", source->ssrc);
for (unsigned i = 0; i < session->ptc; i++)
session->ptv[i].destroy (demux, source->opaque[i]);
block_ChainRelease (source->blocks);
free (source);
}
/**
* Receives an RTP packet and queues it. Not a cancellation point.
*
* @param demux VLC demux object
* @param session RTP session receiving the packet
* @param block RTP packet including the RTP header
*/
void
rtp_queue (demux_t *demux, rtp_session_t *session, block_t *block)
{
demux_sys_t *p_sys = demux->p_sys;
/* RTP header sanity checks (see RFC 3550) */
if (block->i_buffer < 12) //如果RTP包的长度小于12,说明包传输有错误
goto drop;
if ((block->p_buffer[0] >> 6 ) != 2) /* RTP version number(rtp版本号必须为2) */
goto drop;
/* Remove padding if present (判断RTP数据包是否有填充字节,如果有填充字节,解析数据包时,必须将填充字节去掉)*/
if (block->p_buffer[0] & 0x20)
{
uint8_t padding = block->p_buffer[block->i_buffer - 1];
if ((padding == 0) || (block->i_buffer < (12u + padding)))
goto drop; /* illegal value */
block->i_buffer -= padding;
}
mtime_t now = mdate (); //获取精确的时钟信息
rtp_source_t *src = NULL;
const uint16_t seq = rtp_seq (block); //获取序列号
const uint32_t ssrc = GetDWBE (block->p_buffer + 8); //获取SSRC
/* In most case, we know this source already 找到相同的SSRC*/
for (unsigned i = 0, max = session->srcc; i < max; i++)
{
rtp_source_t *tmp = session->srcv[i];
if (tmp->ssrc == ssrc)
{
src = tmp;
break;
}
/* RTP source garbage collection */
if ((tmp->last_rx + p_sys->timeout) < now) //超时了
{
rtp_source_destroy (demux, session, tmp);
if (--session->srcc > 0)
session->srcv[i] = session->srcv[session->srcc - 1]; //将最后一个赋值给删除的那个ssrc
}
}
if (src == NULL) //在原来的会话中没有找到同样的ssrc,说明是新来的ssrc
{
/* New source */
if (session->srcc >= p_sys->max_src) //判断是不是到达了最大的ssrc数的极限值(max)
{
msg_Warn (demux, "too many RTP sessions");
goto drop;
}
rtp_source_t **tab;
tab = realloc (session->srcv, (session->srcc + 1) * sizeof (*tab));
if (tab == NULL)
goto drop;
session->srcv = tab;
src = rtp_source_create (demux, session, ssrc, seq); //创建ssrc
if (src == NULL)
goto drop;
tab[session->srcc++] = src;
/* Cannot compute jitter yet */
}
else
{
const rtp_pt_t *pt = rtp_find_ptype (session, src, block, NULL);
if (pt != NULL)
{
/* Recompute jitter estimate.
* That is computed from the RTP timestamps and the system clock.
* It is independent of RTP sequence. */
uint32_t freq = pt->frequency;
int64_t ts = rtp_timestamp (block);
int64_t d = ((now - src->last_rx) * freq) / CLOCK_FREQ;
d -= ts - src->last_ts;
if (d < 0) d = -d;
src->jitter += ((d - src->jitter) + 8) >> 4;
}
}
src->last_rx = now;
block->i_pts = now; /* store reception time until dequeued */
src->last_ts = rtp_timestamp (block);
/* Check sequence number */
/* NOTE: the sequence number is per-source,
* but is independent from the payload type. */
int16_t delta_seq = seq - src->max_seq;
if ((delta_seq > 0) ? (delta_seq > p_sys->max_dropout)
: (-delta_seq > p_sys->max_misorder))
{
msg_Dbg (demux, "sequence discontinuity"
" (got: %"PRIu16", expected: %"PRIu16")", seq, src->max_seq);
if (seq == src->bad_seq)
{
src->max_seq = src->bad_seq = seq + 1;
src->last_seq = seq - 0x7fffe; /* hack for rtp_decode() */
msg_Warn (demux, "sequence resynchronized");
block_ChainRelease (src->blocks);
src->blocks = NULL;
}
else
{
src->bad_seq = seq + 1;
goto drop;
}
}
else
if (delta_seq >= 0)
src->max_seq = seq + 1;
/* Queues the block in sequence order,
* hence there is a single queue for all payload types. */
block_t **pp = &src->blocks;
for (block_t *prev = *pp; prev != NULL; prev = *pp) //将接收到的数据插入到队列中
{
int16_t delta_seq = seq - rtp_seq (prev);
if (delta_seq < 0)
break;
if (delta_seq == 0)
{
msg_Dbg (demux, "duplicate packet (sequence: %"PRIu16")", seq);
goto drop; /* duplicate */
}
pp = &prev->p_next;
}
block->p_next = *pp;
*pp = block;
/*rtp_decode (demux, session, src);*/
return;
drop:
block_Release (block);
}
vlc调用过程:
/**
* @file input.c
* @brief RTP packet input
*/
/*****************************************************************************
* Copyright 漏 2008 R茅mi Denis-Courmont
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public License
* as published by the Free Software Foundation; either version 2.1
* of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
****************************************************************************/
#ifdef HAVE_CONFIG_H
# include <config.h>
#endif
#include <vlc_common.h>
#include <vlc_demux.h>
#include <vlc_block.h>
#include <vlc_network.h>
#include <unistd.h>
#ifdef HAVE_POLL
# include <poll.h>
#endif
#include "rtp.h"
#ifdef HAVE_SRTP
# include <srtp.h>
#endif
static bool fd_dead (int fd)
{
struct pollfd ufd = { .fd = fd, };
return (poll (&ufd, 1, 0) > 0) && (ufd.revents & POLLHUP);
}
/**
* Gets a datagram from the network.
* @param fd datagram file descriptor
* @return a block or NULL on fatal error (socket dead)
*/
static block_t *rtp_dgram_recv (vlc_object_t *obj, int fd)
{
block_t *block = block_Alloc (0xffff);
ssize_t len;
block_cleanup_push (block);
do
{
len = net_Read (obj, fd, NULL,
block->p_buffer, block->i_buffer, false);
if (((len <= 0) && fd_dead (fd)) || !vlc_object_alive (obj))
{ /* POLLHUP -> permanent (DCCP) socket error */
block_Release (block);
block = NULL;
break;
}
}
while (len == -1);
vlc_cleanup_pop ();
return block ? block_Realloc (block, 0, len) : NULL;
}
/**
* Gets a framed RTP packet.
* @param fd stream file descriptor
* @return a block or NULL in case of fatal error
*/
static block_t *rtp_stream_recv (vlc_object_t *obj, int fd)
{
ssize_t len = 0;
uint8_t hdr[2]; /* frame header */
/* Receives the RTP frame header */
do
{
ssize_t val = net_Read (obj, fd, NULL, hdr + len, 2 - len, false);
if (val <= 0)
return NULL;
len += val;
}
while (len < 2);
block_t *block = block_Alloc (GetWBE (hdr));
/* Receives the RTP packet */
for (ssize_t i = 0; i < len;)
{
ssize_t val;
block_cleanup_push (block);
val = net_Read (obj, fd, NULL,
block->p_buffer + i, block->i_buffer - i, false);
vlc_cleanup_pop ();
if (val <= 0)
{
block_Release (block);
return NULL;
}
i += val;
}
return block;
}
static block_t *rtp_recv (demux_t *demux)
{
demux_sys_t *p_sys = demux->p_sys;
for (block_t *block;; block_Release (block))
{
block = p_sys->framed_rtp
? rtp_stream_recv (VLC_OBJECT (demux), p_sys->fd)
: rtp_dgram_recv (VLC_OBJECT (demux), p_sys->fd);
if (block == NULL)
{
msg_Err (demux, "RTP flow stopped");
break; /* fatal error */
}
if (block->i_buffer < 2)
continue;
/* FIXME */
const uint8_t ptype = rtp_ptype (block);
if (ptype >= 72 && ptype <= 76)
continue; /* Muxed RTCP, ignore for now */
#ifdef HAVE_SRTP
if (p_sys->srtp)
{
size_t len = block->i_buffer;
int canc, err;
canc = vlc_savecancel ();
err = srtp_recv (p_sys->srtp, block->p_buffer, &len);
vlc_restorecancel (canc);
if (err)
{
msg_Dbg (demux, "SRTP authentication/decryption failed");
continue;
}
block->i_buffer = len;
}
#endif
return block; /* success! */
}
return NULL;
}
static void timer_cleanup (void *timer)
{
vlc_timer_destroy ((vlc_timer_t)timer);
}
static void rtp_process (void *data);
void *rtp_thread (void *data)
{
demux_t *demux = data;
demux_sys_t *p_sys = demux->p_sys;
bool autodetect = true;
if (vlc_timer_create (&p_sys->timer, rtp_process, data))
return NULL;
vlc_cleanup_push (timer_cleanup, (void *)p_sys->timer);
for (;;)
{
block_t *block = rtp_recv (demux);
if (block == NULL)
break;
if (autodetect)
{ /* Autodetect payload type, _before_ rtp_queue() */
/* No need for lock - the queue is empty. */
if (rtp_autodetect (demux, p_sys->session, block))
{
block_Release (block);
continue;
}
autodetect = false;
}
int canc = vlc_savecancel ();
vlc_mutex_lock (&p_sys->lock);
rtp_queue (demux, p_sys->session, block);
vlc_mutex_unlock (&p_sys->lock);
vlc_restorecancel (canc);
rtp_process (demux);
}
vlc_cleanup_run ();
return NULL;
}
/**
* Process one RTP packet from the de-jitter queue.
*/
static void rtp_process (void *data)
{
demux_t *demux = data;
demux_sys_t *p_sys = demux->p_sys;
mtime_t deadline;
vlc_mutex_lock (&p_sys->lock);
if (rtp_dequeue (demux, p_sys->session, &deadline))
vlc_timer_schedule (p_sys->timer, true, deadline, 0);
vlc_mutex_unlock (&p_sys->lock);
}