IJKPlayer/FFMPEG 4 Android

1 : native_init

    IjkMediaPlayer_native_init : Do nothing.

2 : native_setup

IjkMediaPlayer_native_setup --> ijkmp_android_create(message_loop)
    A: IjkMediaPlayer *mp = ijkmp_create(msg_loop)
        --> mp->ffplayer = ffp_create()
        --> msg_queue_init(&ffp->msg_queue) : Create message queue
        --> mp->msg_loop = msg_loop --> message_loop_n
        --> ijkmp_get_msg
        -->msg_queue_get(&mp->ffplayer->msg_queue, msg, block)
            (prepared, completed, seek_complete, start, pause, seek)
        --> post_event --> JJKC_IjkMediaPlayer__postEventFromNative
        --> (Java) postEventFromNative
    B: mp->ffplayer->vout = SDL_VoutAndroid_CreateForAndroidSurface();
    C: ffpipeline_create_from_android(mp->ffplayer)
        --> pipeline->func_destroy   = func_destroy;
            pipeline->func_open_video_decoder = func_open_video_decoder
            pipeline->func_open_audio_output  = func_open_audio_output
    D: ffpipeline_set_vout(mp->ffplayer->pipeline, mp->ffplayer->vout)

3: _prepareAsync

--> IjkMediaPlayer_prepareAsync
--> ijkmp_prepare_async
--> ijkmp_prepare_async_l
    A: mp->msg_thread= SDL_CreateThreadEx(&mp->_msg_thread, ijkmp_msg_loop, mp, "ff_msg_loop")
    B: ffp_prepare_async_l
        (1) ffpipeline_open_audio_output --> func_open_audio_output
            --> SDL_AoutAndroid_CreateForOpenSLES
             or 
                SDL_AoutAndroid_CreateForAudioTrack 
                Only get the audio render interface (aout->open_audio, aout->pause_audio, aout->flush_audio, etc)
            --> SDL_AoutSetStereoVolume
        (2) stream_open
            A: frame_queue_init(&is->pictq, &is->videoq, ffp->pictq_size, 1) : **Init video decoded frame buffer queue.**
            B: frame_queue_init(&is->subpq, &is->subtitleq, SUBPICTURE_QUEUE_SIZE, 0) : Init Subtile if needed.
            C: frame_queue_init(&is->sampq, &is->audioq, SAMPLE_QUEUE_SIZE, 1): Init Audio decoded PCM buffer queue.
            D: packet_queue_init(&is->videoq) : Init Video compressed data buffer queue.
            E: packet_queue_init(&is->audioq) : Init Audio compressed data buffer queue.
            F: packet_queue_init(&is->subtitleq) : Init Subtitle compressed data buffer queue if needed.
            G:  init_clock(&is->vidclk, &is->videoq.serial) : Init Video clock
            H: init_clock(&is->audclk, &is->audioq.serial) : Init Audio clock
            I  : init_clock(&is->extclk, &is->extclk.serial) : Init external clock
            J: SDL_CreateThreadEx(&is->_video_refresh_tid, video_refresh_thread, ffp, "ff_vout") --> video_refresh_thread --> video_refresh : Create Video render thread.
            K: SDL_CreateThreadEx(&is->_read_tid, read_thread, ffp, "ff_read") --> read_thread : Create read/input thread.
                1: avformat_alloc_context()
                2: avformat_open_input
                3: avformat_find_stream_info
                4: av_find_best_stream :(Audio/Video/Subtitle)  
                5: stream_component_open(ffp, st_index[AVMEDIA_TYPE_VIDEO])
                    A: avcodec_find_decoder(avctx->codec_id)
                    B: avcodec_open2(avctx, codec, &opts)
                    C: decoder_init(&is->viddec, avctx, &is->videoq, is->continue_read_thread) :Set video compressed buffer queue into created decoder.
                    D: ffpipeline_open_video_decoder 
                                    -- > ffpipenode_create_video_decoder_from_android_mediacodec / ffpipenode_create_video_decoder_from_ffplay
                                        Get decoder interfaces
                    E: decoder_start(&is->viddec, video_thread, ffp, "ff_video_dec")
                    -- > video_thread --> ffpipenode_run_sync 
                    -->  func_run_sync
                    -->  SDL_CreateThreadEx(&opaque->_enqueue_thread, enqueue_thread_func, node, "amediacodec_input_thread")
                    -- > enqueue_thread_func --> feed_input_buffer --> ffp_packet_queue_get_or_buffering : Decoder(MediaCodec) input thread, get compressed data from decoder compressed buffer queue and set into decoder.
                    while (!q->abort_request) -->  : Decoder output thread.
                        drain_output_buffer --> : Drain decoded frame from decoder.
                        ffp_queue_picture --> : Set decoded frame into decoded buffer queue.
                6: stream_component_open(ffp, st_index[AVMEDIA_TYPE_AUDIO])
                    A: codec = avcodec_find_decoder(avctx->codec_id)
                    B: avcodec_open2(avctx, codec, &opts)
                    C: audio_open(ffp, channel_layout, nb_channels, sample_rate, &is->audio_tgt)--> aout_open_audio_n --> Creaete Android_AudioTrack --> start audio_render_thread : aout_thread --> aout_thread_n --> while (!opaque->abort_request)  --> audio_cblk(userdata, buffer, copy_size) : Get decoded PCM data--> Android_AudioTrack_write : Write into Android AudioTrack
                    D: SDL_AoutSetDefaultLatencySeconds
            7: stream_component_open(ffp, st_index[AVMEDIA_TYPE_SUBTITLE])
                                TODO
            8: for (;;) --> Do read/input thread.
                A: is->abort_request : Abort, exit read thread.
                B: is->seek_req : Seek if needed.
                C: av_read_frame(ic, pkt) : Read compressed data.
                D: pb_eof : Deal with EOS.
                E: pkt->stream_index == is->audio_stream --> Put the compressed data into right compressed buffer queue.

4: _start

IjkMediaPlayer_start --> ijkmp_start --> ijkmp_start_l
    --> ffp_notify_msg1(mp->ffplayer, FFP_REQ_START) --> ijkmp_get_msg
    -->FFP_REQ_START--> ffp_start_from_l --> ffp_toggle_buffering 
        --> ffp_seek_to_l
    or
        --> ffp_start_l --> toggle_pause(ffp, 0) --> toggle_pause_l
    -->is->pause_req = pause_on --> set_clock --> is->paused = 0
    -->SDL_AoutPauseAudio ...
    --> ijkmp_change_state_l(mp, MP_STATE_STARTED)

5: _stop

IjkMediaPlayer_stop
    A(1): ijkmp_stop --> ijkmp_stop_l --> ffp_remove_msg --> ffp_stop_l
    --> is->abort_request = 1 --> msg_queue_abort --> q->abort_request = 1
        1: q->abort_request == 1 --> msg_queue_get return -1 --> ijkmp_get_msg return -1 -->  message_loop_n break --> message thread exit
        2: is->abort_request = 1 
            A: video_refresh_thread while (!is->abort_request)
                --> video render exit
            B: read_thread is->abort_request break;
                 --> input thread exit
    B: ijkmp_dec_ref_p --> ijkmp_dec_ref // Not Do This normally 
        1: ijkmp_shutdown --> ijkmp_shutdown_l
            A: ffp_stop_l --> to A(1)
            B: ffp_wait_stop_l --> ffp_stop_l --> A(1) --> stream_close 
                1: packet_queue_abort(&is->videoq)
                    -- > q->abort_request = 1                       
                    A: video_decoder_input_thread : enqueue_thread_func decoder->queue->abort_request  exit (MediaCodec)
                    B: video_decoder_output_thread : func_run_sync decoder->queue->abort_request  exit (MediaCodec)
                2: packet_queue_abort(&is->audioq)
                    --> q->abort_request = 1
                    A: opaque->abort_request
                        --> Audio render thread exit.
                    B: decoder->queue->abort_request
                        --> decoder_decode_frame return -1 
                        --> audio_thread goto the_end
                            --> Audio decoder thread exit

6:_pause

IjkMediaPlayer_pause --> ijkmp_pause --> ijkmp_pause_l
    --> ffp_notify_msg1(mp->ffplayer, FFP_REQ_PAUSE) --> ijkmp_msg_loop
    --> ijkmp_get_msg
        --> msg_queue_get --> FFP_REQ_PAUSE --> ffp_pause_l
        --> toggle_pause(ffp, 1) --> toggle_pause_l --> is->pause_req = 1
        --> stream_toggle_pause_l(ffp, 1)
        --> set_clock 
        --> is->paused = 1
        A: if (is->show_mode != SHOW_MODE_NONE && (!is->paused || is->force_refresh)) : Pause Video render thread
        B: is->paused --> SDL_Delay(10) continue : Pause read thread
        --> SDL_AoutPauseAudio(ffp->aout, pause_on)
            A: SDL_Android_AudioTrack_pause(env, atrack)
            --> aout->pause_audio --> aout_pause_audio
            --> opaque->pause_on = pause_on(true)
            --> while()
            --> if(opaque->pause_on)
            --> SDL_CondWaitTimeout(opaque->wakeup_cond, opaque->wakeup_mutex ,1000)
                **Pause Audio render thread**
            B: sdl_audio_callback --> audio_decode_frame
            --> if (is->paused || is->step) return -1
                : **NOT fill the audio output buffer.**

7 _release

IjkMediaPlayer_release
    A: ijkmp_android_set_surface --> ijkmp_android_set_surface
    --> ijkmp_android_set_surface_l
        1: SDL_VoutAndroid_SetAndroidSurface
            --> SDL_VoutAndroid_SetNativeWindow
            --> SDL_VoutAndroid_SetNativeWindow_l 
            --> SDL_VoutAndroid_invalidateAllBuffers_l
                : Release all output buffers.
            --> vout->opaque->native_window = NULL;
        2: ffpipeline_set_surface --> SDL_VoutAndroid_setAMediaCodec
            --> SDL_VoutAndroid_setAMediaCodec_l
            --> SDL_VoutAndroid_invalidateAllBuffers_l
                : Release all output buffers.
            --> SDL_AMediaCodec_decreaseReferenceP
            --> SDL_AMediaCodec_decreaseReference
            --> if(refCount == 1)SDL_AMediaCodec_delete
            --> *acodec = NULL
            --> vout->opaque->acodec = acodec(NULL)
            --> pipeline->opaque->jsurface = NULL
            --> pipeline->opaque->is_surface_need_reconfigure = true
            --> SDL_JNI_DeleteGlobalRefP(env, &prev_surface)
                : Release pre-surface
    B: ijkmp_shutdown -->ijkmp_shutdown_l
        1: ffp_stop_l --> ...
        2: ffp_wait_stop_l
            A: ffp_stop_l
            B: stream_close
                1: packet_queue_abort(&is->videoq)
                    : Abort video packet queue and
                      effect video decoder thread exit.
                2: packet_queue_abort(&is->audioq)
                    : Abort audio packet queue and
                      effect audio decoder thread exit.
                3: SDL_WaitThread(is->read_tid, NULL)
                    : Read thread exit.
                4: stream_component_close(ffp, is->audio_stream)
                    A: decoder_abort
                        1: packet_queue_abort(d->queue)
                            : Abort audio packet queue and
                             effect video decoder thread exit.
                        2: SDL_WaitThread(d->decoder_tid, NULL)
                            : Exit decoder thread
                        3: packet_queue_flush(d->queue)
                    B: SDL_AoutCloseAudio
                        1: aout->opaque->abort_request = true
                        2: SDL_WaitThread(opaque->audio_tid, NULL)
                            : Exit audio render thread.
                    C: decoder_destroy
                5: stream_component_close(ffp, is->video_stream)
                    A: decoder_abort(&is->viddec, &is->pictq)
                        1: packet_queue_abort(d->queue)
                        2: SDL_WaitThread(d->decoder_tid, NULL)
                            : Exit video decoder thread.
                        3: packet_queue_flush(d->queue)
                    B: decoder_destroy
                6: stream_component_close(ffp, is->subtitle_stream)
                    TODO
                7: avformat_close_input
                8: SDL_WaitThread(is->video_refresh_tid, NULL)
                    : Video render thread exit
                9:  packet_queue_destroy(&is->videoq)
            10: packet_queue_destroy(&is->audioq)
            11: packet_queue_destroy(&is->subtitleq)                                                                                                                                  
            12: frame_queue_destory(&is->pictq)
            13: frame_queue_destory(&is->sampq)
            14: frame_queue_destory(&is->subpq)
        C: jni_set_media_player(env, thiz, NULL)
D: jni_set_media_data_source(env, thiz, NULL)
E: ijkmp_dec_ref_p(&mp)
     ijkmp_dec_ref_p --> ijkmp_dec_ref  // NOT Do this usually. 
        1: ijkmp_shutdown --> ijkmp_shutdown_l
            A: ffp_stop_l --> to A(1)
            B: ffp_wait_stop_l --> ffp_stop_l --> A(1)
                --> stream_close
                1: packet_queue_abort(&is->videoq)
                -- > q->abort_request = 1                       
                    A: video_decoder_input_thread
                        : enqueue_thread_func decoder->
                        queue->abort_request  exit (MediaCodec)
                    B: video_decoder_output_thread
                        : func_run_sync decoder->
                        queue->abort_request  exit (MediaCodec)
                2: packet_queue_abort(&is->audioq)
                  q->abort_request = 1
                    A: opaque->abort_request
                        :Audio render thread exit.
                    B: decoder->queue->abort_request
                    --> decoder_decode_frame return -1
                    --> audio_thread goto the_end
                    --> Audio decoder thread exit

8: _reset

IjkMediaPlayer_reset
    A: IjkMediaPlayer_release(env, thiz);
    B: IjkMediaPlayer_native_setup(env, thiz, weak_thiz);
    C: ijkmp_dec_ref_p(&mp);

9: seekTo

IjkMediaPlayer_seekTo --> ijkmp_seek_to --> ijkmp_seek_to_l
    A: mp->seek_req = 1
    B: mp->seek_msec = msec
    C: ffp_remove_msg(mp->ffplayer, FFP_REQ_SEEK)
    D: ffp_notify_msg2(mp->ffplayer, FFP_REQ_SEEK, (int)msec)
    E: mp->msg_loop = msg_loop --> message_loop_n -->
       ijkmp_get_msg --> msg_queue_get
        case FFP_REQ_SEEK --> ffp_seek_to_l(mp->ffplayer, msg->arg1)
        --> stream_seek(is, seek_pos, 0, 0)
            1: is->seek_pos = pos;   
            2: is->seek_rel = rel;   
            3: is->seek_flags &= ~AVSEEK_FLAG_BYTE;
            4:  if (seek_by_bytes)    
                    is->seek_flags |= AVSEEK_FLAG_BYTE;
            5: is->seek_req = 1; 
    F: read_thread
        if (is->seek_req) --> seek_target = is->seek_pos
        --> ffp_toggle_buffering(ffp, 1) --> avformat_seek_file
            1: packet_queue_flush(&is->audioq)
                --> packet_queue_put(&is->audioq, &flush_pkt)
            2: packet_queue_flush(&is->subtitleq)
                --> packet_queue_put(&is->subtitleq, &flush_pkt)
            3: ffpipenode_flush(ffp->node_vdec)
                --> packet_queue_flush(&is->videoq)
                --> packet_queue_put(&is->videoq, &flush_pkt)
            4: set_clock
            5: ffp_notify_msg1(ffp, FFP_MSG_SEEK_COMPLETE)
            6: ffp_toggle_buffering(ffp, 1)

             Go on read_thread ...