ESP-ADF音频框架 -- stream

ZC_L

已于 2022-10-26 17:47:03 修改

阅读量1.7k

点赞数 1

分类专栏： esp-adf 文章标签：音视频

于 2022-10-26 16:13:39 首次发布

本文链接：https://blog.csdn.net/liang_zhaocong/article/details/127529608

版权

esp-adf 专栏收录该内容

3 篇文章 1 订阅

订阅专栏

由于工作需要，需要ESP32对MP3流进行软解码，为了通过文件进行实时流模拟，研究了ESP-ADF中的流类型。相关流类型及支持的操作如下表所示。

流类型	AUDIO_STREAM_READER `（读类型）`	AUDIO_STREAM_WRITER （写类型）
算法流	Y	N
FatFs 流	Y	Y
HTTP 流	Y	Y
I2S 流	Y	Y
PWM 流	N	Y
原始流	Y	Y
SPIFFS 流	Y	Y
TCP 客户端流	Y	Y
提示音流	Y	N
嵌入式二进制文件流	Y	N
语音合成流	Y	N

思路一

由于MP3流本质上就是二进制数据流，所以以嵌入式二进制流作为前端第一级输入，末端输出采取FatFs流，写入文件中，具体流程如下。

[sdcard] --> 嵌入式二进制文件流 --> mp3_decoder --> fatfs_stream_writer --> [sdcard]

根据应用示例player/pipeline_embed_flash_ton所示，嵌入式二进制文件流在烧录过程中已经将MP3文件写入到flash中，ESP-ADF中没有提供具体的数据流写入API（可能ESP-IDF中有对flash进行写入的API），即没办法对数据进行动态更新，此外嵌入式二进制文件流不支持AUDIO_STREAM_WRITER，所以思路1不可行。

思路二

ESP-ADF框架提供了两种方式对流数据进行处理，分别是基于Pipeline和Element。Pipeline相对于Element，中间变量环节已经由Pipeline进行实现，更加易于管理；而Element更加容易深入了解具体的调用流程，但各个Element之间需要构建缓存空间进行数据交互及输入输出进行重定向。

基于Pipeline方式

基于Element方式

参照示例recorder/element_wav_amr_sdcard及recorder/element_cb_sdcard_amr，为了实现流模拟，通过（fopen，fread）读取sdcard文件数据流，并写入ringbuf1中，定义为ringbuf1的输入，将输出定义为decoder的输入。具体流程如下：

[sdcard] --> FILE --> mp3_decoder --> fatfs_stream_writer --> [sdcard]

相应的测试代码如下：

void app_main() {
    ringbuf_handle_t ringbuf01, ringbuf12;

    audio_element_handle_t mp3_decoder, fatfs_stream_writer;
    ESP_LOGI(TAG, " Create mp3 decoder to decode mp3 file");
    mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG();
    mp3_decoder = mp3_decoder_init(&mp3_cfg);

    ESP_LOGI(TAG, " Create fatfs stream to write data to sdcard");
    fatfs_stream_cfg_t fatfs_cfg_w = FATFS_STREAM_CFG_DEFAULT();
    fatfs_cfg_w.type = AUDIO_STREAM_WRITER;
    fatfs_stream_writer = fatfs_stream_init(&fatfs_cfg_w);

    audio_element_info_t music_info = {0};
    audio_element_getinfo(mp3_decoder, &music_info);
    audio_element_setinfo(fatfs_stream_writer, &music_info);
    audio_element_set_uri(fatfs_stream_writer, "/sdcard/out.wav");

    ringbuf01 = rb_create(RING_BUFFER_SIZE, 1);
    rb_reset(ringbuf01);
//    audio_element_set_output_ringbuf(i2s_stream_reader, ringbuf01);
    audio_element_set_input_ringbuf(mp3_decoder, ringbuf01);

    ringbuf12 = rb_create(RING_BUFFER_SIZE, 1);
    rb_reset(ringbuf12);
    audio_element_set_output_ringbuf(mp3_decoder, ringbuf12);
    audio_element_set_input_ringbuf(fatfs_stream_writer, ringbuf12);

    audio_element_set_event_callback(mp3_decoder, audio_element_event_handler, NULL);
    audio_element_set_event_callback(fatfs_stream_writer, audio_element_event_handler, NULL);

    FILE *fin = fopen("/sdcard/in.mp3", "r");

    int num = 0;
    // memset(buffer, 0, sizeof(buffer));
    char *tmp;
    tmp = (char *)malloc(RING_BUFFER_SIZE);
    int readLen = fread(tmp, sizeof(char), RING_BUFFER_SIZE, fin);
    int ret = rb_write(ringbuf01, tmp, readLen, 0);;
    free(tmp);
    tmp = NULL;
    printf("num: %d, readLen: %d\n", num++, ret);

    audio_element_run(mp3_decoder);
    audio_element_run(fatfs_stream_writer);

    audio_element_resume(mp3_decoder, 0, 0);
    audio_element_resume(fatfs_stream_writer, 0, 0);

    
    while (1)
    {
         memset(buffer, 0, sizeof(buffer));
         readLen = fread(buffer, sizeof(char), RING_BUFFER_SIZE, fin);
         write_file_ringbuf(ringbuf01, buffer, readLen);
         printf("num: %d, readLen: %d\n", num++, readLen);
         if (feof(fin))
         {
             printf("finish\n");
             break;
         }
    }
    
    ESP_LOGI(TAG, "Release all resources");
    audio_element_deinit(fatfs_stream_writer);
    audio_element_deinit(mp3_decoder);

    rb_destroy(ringbuf01);
    rb_destroy(ringbuf12);
}

但是经测试，会导致系统崩溃，还需做进一步验证。

思路三

由于需要获取输入流的控制权，所以该流类型需要支持AUDIO_STREAM_WRITER。对具体的流类型进行逐个排除。

流类型	AUDIO_STREAM_READER `（读类型）`	AUDIO_STREAM_WRITER （写类型）	备注
算法流	Y	N
FatFs 流	Y	Y	没办法控制具体的写入操作
HTTP 流	Y	Y	需要联网
I2S 流	Y	Y	末级输出
PWM 流	N	Y	下一级元素无法获取数据
原始流	Y	Y
SPIFFS 流	Y	Y	没办法控制具体的写入操作
TCP 客户端流	Y	Y	不符合应用场景
提示音流	Y	N	不支持写入操作
嵌入式二进制文件流	Y	N	烧写的时候已固化
语音合成流	Y	N	TTS文本数据

此外原始流提供具体的读写API

int raw_stream_write(audio_element_handle_tpipeline, char *buffer, int buf_size);
int raw_stream_read(audio_element_handle_tpipeline, char *buffer, int buf_size);

static void raw_write_task(void *para)
{
    char *uri = (char *)para;
    char *buf = audio_calloc(1, 4096);
    AUDIO_MEM_CHECK(TAG, buf, return;);

    fatfs_stream_cfg_t fatfs_cfg = FATFS_STREAM_CFG_DEFAULT();
    fatfs_cfg.task_stack = 0;
    fatfs_cfg.type = AUDIO_STREAM_READER;
    audio_element_handle_t fs = fatfs_stream_init(&fatfs_cfg);
    audio_element_set_uri(fs, uri);
    audio_element_process_init(fs);
    audio_element_run(fs);

    ESP_LOGI(TAG, "Raw writing..., URI:%s", uri);
    while (raw_task_run_flag) {
        int ret = audio_element_input(fs, buf, 4096);
        printf(".");
        if (AEL_IO_OK == ret) {
            ESP_LOGE(TAG, "Raw write done");
            audio_player_raw_feed_finish();
            break;
        }
        audio_player_raw_feed_data((uint8_t *)buf, 4096);
    }
    audio_element_process_deinit(fs);
    audio_element_deinit(fs);
    free(buf);
    ESP_LOGI(TAG, "Raw write stop");
    vTaskDelete(NULL);
}

void raw_write(const char *p)
{
    raw_task_run_flag = true;
    if (xTaskCreate(raw_write_task, "RawWriteTask", 3072, (void *)p, 5, NULL) != pdPASS) {
        ESP_LOGE(TAG, "ERROR creating raw_write_task task! Out of memory?");
    }
}

结合上述说明，具体流程如下

[sdcard] --> FILE -->raw_stream_writer--> mp3_decoder --> fatfs_stream_writer --> [sdcard]

具体实现代码如下

static void raw_write_task(void *para)
{
    char *buf = audio_calloc(1, MP3_PACK_SIZE);
    AUDIO_MEM_CHECK(TAG, buf, return;);

    ESP_LOGI(TAG, "Raw writing...\n");
    FILE *fin = fopen("/sdcard/test.mp3", "r");
    while (raw_task_run_flag) {
        int length = fread(buf, 1, MP3_PACK_SIZE, fin);
        printf("length: %d\n", length);
        if (length <= 0) {
            ESP_LOGE(TAG, "Raw write done");
            fclose(fin);
            audio_element_set_ringbuf_done(raw_stream_writer);
            audio_element_finish_state(raw_stream_writer);
            break;
        }
        raw_stream_write(raw_stream_writer, buf, length);
//        audio_player_raw_feed_data((uint8_t *)buf, 4096);
    }
    free(buf);
    ESP_LOGI(TAG, "Raw write stop");
    vTaskDelete(NULL);
}

void app_main(void)
{

    audio_pipeline_cfg_t pipeline_cfg = DEFAULT_AUDIO_PIPELINE_CONFIG();
    audio_pipeline_handle_t pipeline = audio_pipeline_init(&pipeline_cfg);
    mem_assert(pipeline);

    raw_stream_cfg_t raw_cfg = RAW_STREAM_CFG_DEFAULT();
    raw_cfg.type = AUDIO_STREAM_WRITER;
    raw_stream_writer = raw_stream_init(&raw_cfg);

    ESP_LOGI(TAG, " Create mp3 decoder to decode mp3 file");
    mp3_decoder_cfg_t mp3_cfg = DEFAULT_MP3_DECODER_CONFIG();
    mp3_decoder = mp3_decoder_init(&mp3_cfg);

    ESP_LOGI(TAG, " Create fatfs stream to write data to sdcard");
    fatfs_stream_cfg_t fatfs_cfg_w = FATFS_STREAM_CFG_DEFAULT();
    fatfs_cfg_w.type = AUDIO_STREAM_WRITER;
    fatfs_stream_writer = fatfs_stream_init(&fatfs_cfg_w);

    ESP_LOGI(TAG, "[2.3] Register all elements to audio pipeline");
    audio_pipeline_register(pipeline, raw_stream_writer, "raw_writer");
    audio_pipeline_register(pipeline, mp3_decoder, "mp3");
    audio_pipeline_register(pipeline, fatfs_stream_writer, "fatfs_writer");

    ESP_LOGI(TAG, "[2.4] Link it together raw_stream_writer-->mp3_decoder-->fatfs_stream_writer-->[sdcard]");
    const char *link_tag[3] = {"raw_writer","mp3", "fatfs_writer"};
    audio_pipeline_link(pipeline, &link_tag[0], 3);
    

    ESP_LOGI(TAG, "[ 4 ] Set up  event listener");
    audio_event_iface_cfg_t evt_cfg = AUDIO_EVENT_IFACE_DEFAULT_CFG();
    audio_event_iface_handle_t evt = audio_event_iface_init(&evt_cfg);

    ESP_LOGI(TAG, "[4.1] Listening event from all elements of pipeline");
    audio_pipeline_set_listener(pipeline, evt);

    raw_task_run_flag = true;
    if (xTaskCreate(raw_write_task, "RawWriteTask", 3072, NULL, 5, NULL) != pdPASS) {
        ESP_LOGE(TAG, "ERROR creating raw_write_task task! Out of memory?");
    }

    audio_element_set_uri(fatfs_stream_writer, "/sdcard/out.wav");

    audio_pipeline_run(pipeline);

    while (1) {
        audio_event_iface_msg_t msg;
        esp_err_t ret = audio_event_iface_listen(evt, &msg, portMAX_DELAY);
        if (ret != ESP_OK) {
            continue;
        }

        if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT) {
            // Set music info for a new song to be played
            if (msg.source == (void *) mp3_decoder
                && msg.cmd == AEL_MSG_CMD_REPORT_MUSIC_INFO) {
                audio_element_info_t music_info = {0};
                audio_element_getinfo(mp3_decoder, &music_info);
                ESP_LOGI(TAG, "[ * ] Received music info from mp3 decoder, sample_rates=%d, bits=%d, ch=%d, duration=%d",
                         music_info.sample_rates, music_info.bits, music_info.channels, music_info.duration);
                audio_element_setinfo(fatfs_stream_writer, &music_info);
                
                continue;
            }

                /* Stop when the last pipeline element (fatfs_stream_writer in this case) receives stop event */
            if (msg.source_type == AUDIO_ELEMENT_TYPE_ELEMENT && msg.source == (void *) fatfs_stream_writer
                && msg.cmd == AEL_MSG_CMD_REPORT_STATUS
                && (((int)msg.data == AEL_STATUS_STATE_STOPPED) || ((int)msg.data == AEL_STATUS_STATE_FINISHED)
                    || ((int)msg.data == AEL_STATUS_ERROR_OPEN))) {
                ESP_LOGW(TAG, "[ * ] Stop event received");
                break;
            }
        }

    }


    ESP_LOGI(TAG, "[ 6 ] Stop audio_pipeline");
    audio_pipeline_stop(pipeline);
    audio_pipeline_wait_for_stop(pipeline);
    audio_pipeline_terminate(pipeline);
    audio_pipeline_unregister(pipeline, raw_stream_writer);
    audio_pipeline_unregister(pipeline, mp3_decoder);
    audio_pipeline_unregister(pipeline, fatfs_stream_writer);

    /* Terminate the pipeline before removing the listener */
    audio_pipeline_remove_listener(pipeline);

    /* Make sure audio_pipeline_remove_listener is called before destroying event_iface */
    audio_event_iface_destroy(evt);

    /* Release all resources */
    audio_pipeline_deinit(pipeline);
    audio_element_deinit(raw_stream_writer);
    audio_element_deinit(mp3_decoder);
    audio_element_deinit(fatfs_stream_writer);
    
}

经测试，可以实现解码，输入流可控，符合具体需求。