pjmedia系列之媒体设备pjmedia_snd_port

在simpleua.c文件，当协商成功call_on_media_update中，会创建音频设备对象。

static pjmedia_snd_port	    *g_snd_port;    /* Sound device.		*/

static void call_on_media_update( pjsip_inv_session *inv, pj_status_t status)
{}
    pjmedia_port *media_port;
    
    /* Get the media port interface of the audio stream. 
     * Media port interface is basicly a struct containing get_frame() and
     * put_frame() function. With this media port interface, we can attach
     * the port interface to conference bridge, or directly to a sound
     * player/recorder device.
     */
    pjmedia_stream_get_port(g_med_stream, &media_port);

    /* Create sound port */
    pjmedia_snd_port_create(inv->pool,
                            PJMEDIA_AUD_DEFAULT_CAPTURE_DEV,
                            PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV,
                            PJMEDIA_PIA_SRATE(&media_port->info),/* clock rate	    */
                            PJMEDIA_PIA_CCNT(&media_port->info),/* channel count    */
                            PJMEDIA_PIA_SPF(&media_port->info), /* samples per frame*/
                            PJMEDIA_PIA_BITS(&media_port->info),/* bits per sample  */
                            0,
                            &g_snd_port);


    status = pjmedia_snd_port_connect(g_snd_port, media_port);
}

首先从stream获取一个media_port的接口实例，这个实例是在创建流的时候创建的。

PJ_DEF(pj_status_t) pjmedia_stream_get_port( pjmedia_stream *stream,
					     pjmedia_port **p_port )
{
    *p_port = &stream->port;
    return PJ_SUCCESS;
}
   
pjmedia_stream_create()
{
...
	stream->port.put_frame = &put_frame;
	stream->port.get_frame = &get_frame;
...
}

这里实际上是把stream的两个回调，设置到音频设备，当设备mic采集到数据时，最终调用put_frame，当要播放数据时，调用get_frame获取。stream中的两个回调稍后再分析，继续看音频设备。

结构体

struct pjmedia_snd_port
{
    int			 rec_id;
    int			 play_id;
    pj_uint32_t		 aud_caps;
    pjmedia_aud_param	 aud_param;
    pjmedia_aud_stream	*aud_stream;
    pjmedia_dir		 dir;
    pjmedia_port	*port;

    pjmedia_clock_src    cap_clocksrc,
                         play_clocksrc;

    unsigned		 clock_rate;
    unsigned		 channel_count;
    unsigned		 samples_per_frame;
    unsigned		 bits_per_sample;
    unsigned		 options;
    unsigned		 prm_ec_options;


    /* audio frame preview callbacks */
    void		*user_data;
    pjmedia_aud_play_cb  on_play_frame;
    pjmedia_aud_rec_cb   on_rec_frame;
};

结构体里有两个回到函数指针on_play_frame、on_rec_frame，会指向前面讲的stream两个函数put_frame和get_frame。 

创建

PJ_DEF(pj_status_t) pjmedia_snd_port_create( pj_pool_t *pool,
					     int rec_id,
					     int play_id,
					     unsigned clock_rate,
					     unsigned channel_count,
					     unsigned samples_per_frame,
					     unsigned bits_per_sample,
					     unsigned options,
					     pjmedia_snd_port **p_port)
{
    pjmedia_snd_port_param param;
    pj_status_t status;

    pjmedia_snd_port_param_default(&param);

    /* Normalize rec_id & play_id */
    if (rec_id < 0)
	rec_id = PJMEDIA_AUD_DEFAULT_CAPTURE_DEV;
    if (play_id < 0)
	play_id = PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV;

    status = pjmedia_aud_dev_default_param(rec_id, &param.base);
    if (status != PJ_SUCCESS)
	return status;

    param.base.dir = PJMEDIA_DIR_CAPTURE_PLAYBACK;
    param.base.rec_id = rec_id;
    param.base.play_id = play_id;
    param.base.clock_rate = clock_rate;
    param.base.channel_count = channel_count;
    param.base.samples_per_frame = samples_per_frame;
    param.base.bits_per_sample = bits_per_sample;
    param.options = options;
    param.ec_options = 0;

    return pjmedia_snd_port_create2(pool, &param, p_port);
}

初始化一些参数，然后调用 pjmedia_snd_port_create2。

/*
 * Create sound port.
 */
PJ_DEF(pj_status_t) pjmedia_snd_port_create2(pj_pool_t *pool,
					     const pjmedia_snd_port_param *prm,
					     pjmedia_snd_port **p_port)
{
    pjmedia_snd_port *snd_port;
    pj_status_t status;
    unsigned ptime_usec;

    PJ_ASSERT_RETURN(pool && prm && p_port, PJ_EINVAL);

    snd_port = PJ_POOL_ZALLOC_T(pool, pjmedia_snd_port);
    PJ_ASSERT_RETURN(snd_port, PJ_ENOMEM);

    snd_port->dir = prm->base.dir;
    snd_port->rec_id = prm->base.rec_id;
    snd_port->play_id = prm->base.play_id;
    snd_port->clock_rate = prm->base.clock_rate;
    snd_port->channel_count = prm->base.channel_count;
    snd_port->samples_per_frame = prm->base.samples_per_frame;
    snd_port->bits_per_sample = prm->base.bits_per_sample;
    pj_memcpy(&snd_port->aud_param, &prm->base, sizeof(snd_port->aud_param));
    snd_port->options = prm->options;
    snd_port->prm_ec_options = prm->ec_options;
    snd_port->user_data = prm->user_data;
    snd_port->on_play_frame = prm->on_play_frame;
    snd_port->on_rec_frame = prm->on_rec_frame;

    ptime_usec = prm->base.samples_per_frame * 1000 / prm->base.channel_count /
                 prm->base.clock_rate * 1000;
    pjmedia_clock_src_init(&snd_port->cap_clocksrc, PJMEDIA_TYPE_AUDIO,
                           snd_port->clock_rate, ptime_usec);
    pjmedia_clock_src_init(&snd_port->play_clocksrc, PJMEDIA_TYPE_AUDIO,
                           snd_port->clock_rate, ptime_usec);
    
    /* Start sound device immediately.
     * If there's no port connected, the sound callback will return
     * empty signal.
     */
    status = start_sound_device( pool, snd_port );
    if (status != PJ_SUCCESS) {
	pjmedia_snd_port_destroy(snd_port);
	return status;
    }

    *p_port = snd_port;
    return PJ_SUCCESS;
}

 pjmedia_snd_port_create2同样是初始化一些参数，主要有时钟频率，最后调用start_sound_device。

static pj_status_t start_sound_device( pj_pool_t *pool,
				       pjmedia_snd_port *snd_port )
{
    pjmedia_aud_rec_cb snd_rec_cb;
    pjmedia_aud_play_cb snd_play_cb;
    pjmedia_aud_param param_copy;
    pj_status_t status;

...

    /* Use different callback if format is not PCM */
    if (snd_port->aud_param.ext_fmt.id == PJMEDIA_FORMAT_L16) {
	snd_rec_cb = &rec_cb;
	snd_play_cb = &play_cb;
    } else {
	snd_rec_cb = &rec_cb_ext;
	snd_play_cb = &play_cb_ext;
    }

    /* Open the device */
    status = pjmedia_aud_stream_create(&param_copy,
				       snd_rec_cb,
				       snd_play_cb,
				       snd_port,
				       &snd_port->aud_stream);

...
    /* Start sound stream. */
    if (!(snd_port->options & PJMEDIA_SND_PORT_NO_AUTO_START)) {
	status = pjmedia_aud_stream_start(snd_port->aud_stream);
    }


    return PJ_SUCCESS;
}

 由流程可知，真正到pjmedia_aud_stream_create函数才创建设备，并且传入两个回调rec_cb和play_cb，这两个回调并不是stream的那两个回调，而是sound_port.c这一层自己实现的回调，play_cb里面才会调用stream的回调，也就是多了一层，这点跟transport是一样的，都是在本层实现一个回调，然后在调用上层设置的回调。

pjmedia_aud_stream_create的实现在pjmedia/audiodev.c（注意不是pjmedia-audiodev/audiodev.c）。

/* API: Open audio stream object using the specified parameters. */
PJ_DEF(pj_status_t) pjmedia_aud_stream_create(const pjmedia_aud_param *prm,
					      pjmedia_aud_rec_cb rec_cb,
					      pjmedia_aud_play_cb play_cb,
					      void *user_data,
					      pjmedia_aud_stream **p_aud_strm)
{
    pjmedia_aud_dev_factory *rec_f=NULL, *play_f=NULL, *f=NULL;
    pjmedia_aud_param param;
    pj_status_t status;

    PJ_ASSERT_RETURN(prm && prm->dir && p_aud_strm, PJ_EINVAL);
    PJ_ASSERT_RETURN(aud_subsys.pf, PJMEDIA_EAUD_INIT);
    PJ_ASSERT_RETURN(prm->dir==PJMEDIA_DIR_CAPTURE ||
		     prm->dir==PJMEDIA_DIR_PLAYBACK ||
		     prm->dir==PJMEDIA_DIR_CAPTURE_PLAYBACK,
		     PJ_EINVAL);


	status = lookup_dev(param.rec_id, &rec_f, &index);

	status = lookup_dev(param.play_id, &play_f, &index);


    /* Create the stream */
    status = f->op->create_stream(f, &param, rec_cb, play_cb,
				  user_data, p_aud_strm);
    if (status != PJ_SUCCESS)
	return status;

    /* Assign factory id to the stream */
    (*p_aud_strm)->sys.drv_idx = f->sys.drv_idx;
    return PJ_SUCCESS;
}

先通过lookup_dev搜索设备工厂，然后通过工厂创建设备f->op->create_stream。关键点来了，如何搜索到设备。

/* Internal: lookup device id */
static pj_status_t lookup_dev(pjmedia_aud_dev_index id,
			      pjmedia_aud_dev_factory **p_f,
			      unsigned *p_local_index)
{
    int f_id, index;

    if (id < 0) {
	unsigned i;

	if (id == PJMEDIA_AUD_INVALID_DEV)
	    return PJMEDIA_EAUD_INVDEV;

	for (i=0; i<aud_subsys.drv_cnt; ++i) {
	    pjmedia_aud_driver *drv = &aud_subsys.drv[i];
	    if (drv->dev_idx >= 0) {
		id = drv->dev_idx;
		make_global_index(i, &id);
		break;
	    } else if (id==PJMEDIA_AUD_DEFAULT_CAPTURE_DEV && 
		drv->rec_dev_idx >= 0) 
	    {
		id = drv->rec_dev_idx;
		make_global_index(i, &id);
		break;
	    } else if (id==PJMEDIA_AUD_DEFAULT_PLAYBACK_DEV && 
		drv->play_dev_idx >= 0) 
	    {
		id = drv->play_dev_idx;
		make_global_index(i, &id);
		break;
	    }
	}

	if (id < 0) {
	    return PJMEDIA_EAUD_NODEFDEV;
	}
    }

    f_id = GET_FID(aud_subsys.dev_list[id]);
    index = GET_INDEX(aud_subsys.dev_list[id]);

    if (f_id < 0 || f_id >= (int)aud_subsys.drv_cnt)
	return PJMEDIA_EAUD_INVDEV;

    if (index < 0 || index >= (int)aud_subsys.drv[f_id].dev_cnt)
	return PJMEDIA_EAUD_INVDEV;

    *p_f = aud_subsys.drv[f_id].f;
    *p_local_index = (unsigned)index;

    return PJ_SUCCESS;

}

从这里可以看出，全局变量aud_subsys已经存储了所有的音频设备，这里把它找出来即可。那音频设备是不是在初始化的时候就搜索完了呢？aud_subsys全局变量什么时候赋值的？继续分析

在pjmedia-audiodev/audiodev.c中，有个音频子系统初始化函数

PJ_DEF(pj_status_t) pjmedia_aud_subsys_init(pj_pool_factory *pf)
{
    unsigned i;
    pj_status_t status;
    pjmedia_aud_subsys *aud_subsys = pjmedia_get_aud_subsys();

    /* Allow init() to be called multiple times as long as there is matching
     * number of shutdown().
     */
    if (aud_subsys->init_count++ != 0) {
	return PJ_SUCCESS;
    }

    /* Register error subsystem */
    status = pj_register_strerror(PJMEDIA_AUDIODEV_ERRNO_START,
				  PJ_ERRNO_SPACE_SIZE,
				  &pjmedia_audiodev_strerror);
    pj_assert(status == PJ_SUCCESS);

    /* Init */
    aud_subsys->pf = pf;
    aud_subsys->drv_cnt = 0;
    aud_subsys->dev_cnt = 0;

    /* Register creation functions */
#if PJMEDIA_AUDIO_DEV_HAS_OPENSL
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_opensl_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_ANDROID_JNI
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_android_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_BB10
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_bb10_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_ALSA
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_alsa_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_COREAUDIO
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_coreaudio_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_PORTAUDIO
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_pa_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_WMME
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_wmme_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_BDIMAD
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_bdimad_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_VAS
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_symb_vas_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_APS
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_aps_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_SYMB_MDA
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_symb_mda_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_WASAPI
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_wasapi_factory;
#endif
#if PJMEDIA_AUDIO_DEV_HAS_NULL_AUDIO
    aud_subsys->drv[aud_subsys->drv_cnt++].create = &pjmedia_null_audio_factory;
#endif

    /* Initialize each factory and build the device ID list */
    for (i=0; i<aud_subsys->drv_cnt; ++i) {
	status = pjmedia_aud_driver_init(i, PJ_FALSE);
	if (status != PJ_SUCCESS) {
	    pjmedia_aud_driver_deinit(i);
	    continue;
	}
    }

    return aud_subsys->dev_cnt ? PJ_SUCCESS : status;
}

也就是说，初始化的时候调用这个函数，就会根据根据编译宏，把各种音频类型工厂添加到子系统全局变量，其中我们关注pjmedia_alsa_factory。而音频子系统的初始化，是在创建媒体端点的时候。

PJ_INLINE(pj_status_t) pjmedia_endpt_create(pj_pool_factory *pf,
					    pj_ioqueue_t *ioqueue,
					    unsigned worker_cnt,
					    pjmedia_endpt **p_endpt)
{
    /* This function is inlined to avoid build problem due to circular
     * dependency, i.e: this function prevents pjmedia's dependency on
     * pjmedia-audiodev.
     */

    pj_status_t status;

    /* Sound */
    status = pjmedia_aud_subsys_init(pf);
    if (status != PJ_SUCCESS)
        return status;

    status = pjmedia_endpt_create2(pf, ioqueue, worker_cnt, p_endpt);
    if (status != PJ_SUCCESS) {
        pjmedia_aud_subsys_shutdown();
    }
    
    return status;
}

这样分析后，整个脉络就理清了 。初始化的时候创建媒体端点pjmedia_endpt，同时初始化了音频子系统，把各种类型的音频设备工厂添加到全局变量static pjmedia_aud_subsys aud_subsys;。这样当创建设备时，就可以遍历这些工厂，寻找合适的工厂，通过工厂创建设备实例。比如alsa类型的设备在alsa_dev.c

static pjmedia_aud_dev_factory_op alsa_factory_op =
{
    &alsa_factory_init,
    &alsa_factory_destroy,
    &alsa_factory_get_dev_count,
    &alsa_factory_get_dev_info,
    &alsa_factory_default_param,
    &alsa_factory_create_stream,
    &alsa_factory_refresh
};

到此，设备的创建流程基本分析完了。音频设备分三层，最底层的是各种设备类型，比如alsa，再上一层是抽象设备操作audiodev.c，最上面一层是设备接口sound_port。

数据流

创建完设备，我们再来分析数据流向，以alsa为例，在alsa_dev.c中，会创建播放和采集两条线程。

static pj_status_t alsa_stream_start (pjmedia_aud_stream *s)
{
    struct alsa_stream *stream = (struct alsa_stream*)s;
    pj_status_t status = PJ_SUCCESS;

    stream->quit = 0;
    if (stream->param.dir & PJMEDIA_DIR_PLAYBACK) {
	status = pj_thread_create (stream->pool,
				   "alsasound_playback",
				   pb_thread_func,
				   stream,
				   0, //ZERO,
				   0,
				   &stream->pb_thread);


    if (stream->param.dir & PJMEDIA_DIR_CAPTURE) {
	status = pj_thread_create (stream->pool,
				   "alsasound_playback",
				   ca_thread_func,
				   stream,
				   0, //ZERO,
				   0,
				   &stream->ca_thread);

    }

    return status;
}

以播放线程为例，采集线程一样。

static int pb_thread_func (void *arg)
{
    struct alsa_stream* stream = (struct alsa_stream*) arg;
    snd_pcm_t* pcm             = stream->pb_pcm;
    int size                   = stream->pb_buf_size;
    snd_pcm_uframes_t nframes  = stream->pb_frames;
    void* user_data            = stream->user_data;
    char* buf 		       = stream->pb_buf;
    pj_timestamp tstamp;
    int result;

    pj_bzero (buf, size);
    tstamp.u64 = 0;


    snd_pcm_prepare (pcm);

    while (!stream->quit) {
	pjmedia_frame frame;

	frame.type = PJMEDIA_FRAME_TYPE_AUDIO;
	frame.buf = buf;
	frame.size = size;
	frame.timestamp.u64 = tstamp.u64;
	frame.bit_info = 0;

	result = stream->pb_cb (user_data, &frame);
	if (result != PJ_SUCCESS || stream->quit)
	    break;

	if (frame.type != PJMEDIA_FRAME_TYPE_AUDIO)
	    pj_bzero (buf, size);

	result = snd_pcm_writei (pcm, buf, nframes);
	if (result == -EPIPE) {
	    PJ_LOG (4,(THIS_FILE, "pb_thread_func: underrun!"));
	    snd_pcm_prepare (pcm);
	} else if (result < 0) {
	    PJ_LOG (4,(THIS_FILE, "pb_thread_func: error writing data!"));
	}

	tstamp.u64 += nframes;
    }

    snd_pcm_drain (pcm);
    TRACE_((THIS_FILE, "pb_thread_func: Stopped"));
    return PJ_SUCCESS;
}

播放线程先通过回调拿到待播放的音频数据stream->pb_cb ，然后写到声卡snd_pcm_writei。pb_cb就是sound_port.c中的play_cb，来看下play_cb的流程。

static pj_status_t play_cb(void *user_data, pjmedia_frame *frame)
{
    pjmedia_snd_port *snd_port = (pjmedia_snd_port*) user_data;
    pjmedia_port *port;
    const unsigned required_size = (unsigned)frame->size;
    pj_status_t status;

    port = snd_port->port;
    status = pjmedia_port_get_frame(port, frame);

    /* Invoke preview callback */
    if (snd_port->on_play_frame)
	(*snd_port->on_play_frame)(snd_port->user_data, frame);

    return PJ_SUCCESS;
}

play_cb做了两件事

1、通过pjmedia_port* port获取一帧数据

port是snd_port的成员，这个成员是在什么时候赋值的？回到simpleua.c中，pjmedia_snd_port_create后，还调用了一个函数pjmedia_snd_port_connect(g_snd_port, media_port);把stream的media_port传进去。

PJ_DEF(pj_status_t) pjmedia_snd_port_connect( pjmedia_snd_port *snd_port,
					      pjmedia_port *port)
{
    pjmedia_audio_format_detail *afd;

 ...

    /* Port is okay. */
    snd_port->port = port;
    return PJ_SUCCESS;
}

/**
 * Get a frame from the port (and subsequent downstream ports).
 */
PJ_DEF(pj_status_t) pjmedia_port_get_frame( pjmedia_port *port,
					    pjmedia_frame *frame )
{
    PJ_ASSERT_RETURN(port && frame, PJ_EINVAL);

    if (port->get_frame)
	return port->get_frame(port, frame);
    else {
	frame->type = PJMEDIA_FRAME_TYPE_NONE;
	return PJ_EINVALIDOP;
    }
}

 所以这个port->get_frame就是stream.c中的get_frame。

2、通过pjmedia_snd_port调用预览，实际上跟踪会发现，预览指针并没有赋值。

调用pjmedia_snd_port* snd_port的on_play_frame函数指针，通过user_data传递snd_port。从pjmedia_aud_stream_create函数可以知道，第4个参数就是user_data，这个参数是在start_sound_device传入的第二个参数pjmedia_snd_port *snd_port。跟踪发现，想创建设备的时候，为on_play_frame指针赋值为空，那什么时候有值？

这样整个播放数据流就清楚了。首先在alsa创建播放线程，播放线程通过一系列回调，从stream拿到一帧数据，然后写设备播放，关键是捋清楚多层回调的关系。