RK3399 Android 10 Camera2保存录像时缩略图获取为空

RK3399 Android 10相机录像保存时无法获取缩略预览图

• 先找到录像点击按钮

//点击快门按钮时可以通过log打印看到停止录像的流程

• onShutterButtonClick()

//这里主要看停止的流程即stop = true时会进入onStopVideoRecording方法
public void onShutterButtonClick() {
        Log.d(TAG, "onShutterButtonClick");
        if (mSwitchingCamera || !mAppController.getCameraAppUI().isModeCoverHide()) {
            Log.d(TAG," mSwitchingCamera or isModeCoverHide not record!!!!!");
            return;
        }
        boolean stop = mMediaRecorderRecording;

        if (stop) {
            long delta = SystemClock.uptimeMillis() - mRecordingStartTime;
            Log.i(TAG, "mRecordingStartTime = " + mRecordingStartTime + ", delta = " + delta);
            if (delta < 1500) {
                Log.i(TAG, "recorder duration too short");
                return;
            }
            // CameraAppUI mishandles mode option enable/disable
            // for video, override that
            mAppController.getCameraAppUI().enableModeOptions();
            onStopVideoRecording();
        } else {
            int countDownDuration = mActivity.getSettingsManager()
                .getInteger(SettingsManager.SCOPE_GLOBAL, Keys.KEY_COUNTDOWN_DURATION);
            mTimerDuration = countDownDuration;
            if (countDownDuration > 0) {
                // Start count down.
                mAppController.getCameraAppUI().transitionToCancel();
                mAppController.getCameraAppUI().hideModeOptions();
                mUI.startCountdown(countDownDuration);
                return;
            } else {
                // CameraAppUI mishandles mode option enable/disable
                // for video, override that
                mAppController.getCameraAppUI().disableModeOptions();
                startVideoRecording();
            }
        }
        mAppController.setShutterEnabled(false);
        if (mCameraSettings != null) {
            mFocusManager.onShutterUp(mCameraSettings.getCurrentFocusMode());
        }

        // Keep the shutter button disabled when in video capture intent
        // mode and recording is stopped. It'll be re-enabled when
        // re-take button is clicked.
        if (!(mIsVideoCaptureIntent && stop)) {
            mHandler.sendEmptyMessageDelayed(MSG_ENABLE_SHUTTER_BUTTON, SHUTTER_BUTTON_TIMEOUT);
        }
    }

• onStopVideoRecording()

//这里可以看到有调用stopVideoRecording方法
private void onStopVideoRecording() {
        AudioManager am = (AudioManager) mActivity.getSystemService(Context.AUDIO_SERVICE);
        am.abandonAudioFocus(null);
        mAppController.getCameraAppUI().setSwipeEnabled(true);
        boolean recordFail = stopVideoRecording();
        if (mIsVideoCaptureIntent) {
            if (mQuickCapture) {
                doReturnToCaller(!recordFail);
            } else if (!recordFail) {
                showCaptureResult();
            }
            mAppController.getCameraAppUI().disableModeOptions();
        } else if (!recordFail){
            // Start capture animation.
            if (!mPaused && ApiHelper.HAS_SURFACE_TEXTURE_RECORDING) {
                // The capture animation is disabled on ICS because we use SurfaceView
                // for preview during recording. When the recording is done, we switch
                // back to use SurfaceTexture for preview and we need to stop then start
                // the preview. This will cause the preview flicker since the preview
                // will not be continuous for a short period of time.
                //mAppController.startFlashAnimation(false);
            }
        }
    }

• stopVideoRecording()

private boolean stopVideoRecording() {
        // Do nothing if camera device is still capturing photo. Monkey test can trigger app crashes
        // (b/17313985) without this check. Crash could also be reproduced by continuously tapping
        // on shutter button and preview with two fingers.
        if (mSnapshotInProgress) {
            Log.v(TAG, "Skip stopVideoRecording since snapshot in progress");
            return true;
        }
        Log.v(TAG, "stopVideoRecording");

        // Re-enable sound as early as possible to avoid interfering with stop
        // recording sound.
        restoreRingerMode();

        mUI.setSwipingEnabled(true);
        mUI.showPassiveFocusIndicator();
        mAppController.getCameraAppUI().setShouldSuppressCaptureIndicator(false);

        boolean fail = false;
    	//因为mMediaRecorderRecording等于true,所以会走入下面的判断
        if (mMediaRecorderRecording) {
            boolean shouldAddToMediaStoreNow = false;

            try {
                mMediaRecorder.setOnErrorListener(null);
                mMediaRecorder.setOnInfoListener(null);
                mMediaRecorder.stop();
                //这里将shouldAddToMediaStoreNow设置为true
                shouldAddToMediaStoreNow = true;
                mCurrentVideoFilename = mVideoFilename;
                Log.v(TAG, "stopVideoRecording: current video filename: " + mCurrentVideoFilename);
            } catch (RuntimeException e) {
                Log.e(TAG, "stop fail",  e);
                if (mVideoFilename != null) {
                    deleteVideoFile(mVideoFilename);
                }
                fail = true;
            }
            mMediaRecorderRecording = false;
            mActivity.unlockOrientation();

            // If the activity is paused, this means activity is interrupted
            // during recording. Release the camera as soon as possible because
            // face unlock or other applications may need to use the camera.
            if (mPaused) {
                // b/16300704: Monkey is fast so it could pause the module while recording.
                // stopPreview should definitely be called before switching off.
                stopPreview();
                closeCamera();
            }

            mUI.showRecordingUI(false);
            // The orientation was fixed during video recording. Now make it
            // reflect the device orientation as video recording is stopped.
            mUI.setOrientationIndicator(0, true);
            mActivity.enableKeepScreenOn(false);
            //上面有将shouldAddToMediaStoreNow设置为true 同事fail默认为false,因此会走入下面的if语句
            if (shouldAddToMediaStoreNow && !fail) {
                //然后这里进入saveVideo方法
                if (mVideoFileDescriptor == null) {
                    saveVideo();
                } else if (mIsVideoCaptureIntent) {
                    // if no file save is needed, we can show the post capture UI now
                    showCaptureResult();
                }
            }
        }
        // release media recorder
        releaseMediaRecorder();

        mAppController.getCameraAppUI().showModeOptions();
        mAppController.getCameraAppUI().animateBottomBarToFullSize(mShutterIconId);
        if (!mPaused && mCameraDevice != null) {
            setFocusParameters();
            mCameraDevice.lock();
            if (!ApiHelper.HAS_SURFACE_TEXTURE_RECORDING) {
                stopPreview();
                // Switch back to use SurfaceTexture for preview.
                startPreview();
            }
            // Update the parameters here because the parameters might have been altered
            // by MediaRecorder.
            mCameraSettings = mCameraDevice.getSettings();
        }

        // Check this in advance of each shot so we don't add to shutter
        // latency. It's true that someone else could write to the SD card
        // in the mean time and fill it, but that could have happened
        // between the shutter press and saving the file too.
        mActivity.updateStorageSpaceAndHint(null);

        return fail;
    }

• saveVideo()里面是通过MediaSaver接口的实现类MediaSaverImpI.addVideo方法

private void saveVideo() {
        if (mVideoFileDescriptor == null) {
            long duration = SystemClock.uptimeMillis() - mRecordingStartTime;
            if (duration > 0) {
                //
            } else {
                Log.w(TAG, "Video duration <= 0 : " + duration);
            }
            mCurrentVideoValues.put(Video.Media.SIZE, new File(mCurrentVideoFilename).length());
            mCurrentVideoValues.put(Video.Media.DURATION, duration);
            getServices().getMediaSaver().addVideo(mCurrentVideoFilename,
                    mCurrentVideoValues, mOnVideoSavedListener);
            logVideoCapture(duration);
        }
        mCurrentVideoValues = null;
    }

• 然后addVideo方法中new 了一个 AsyncTask

@Override
public void addVideo(String path, ContentValues values, OnMediaSavedListener l) {
    // We don't set a queue limit for video saving because the file
    // is already in the storage. Only updating the database.
    new VideoSaveTask(path, values, l, mContentResolver).execute();
}
//VideoSaveTask在onPostExecute方法将uri通过构造函数中的OnMediaSavedListener即mOnVideoSavedListener将其传递出去
private final MediaSaver.OnMediaSavedListener mOnVideoSavedListener =
            new MediaSaver.OnMediaSavedListener() {
                @Override
                public void onMediaSaved(Uri uri) {
                    if (uri != null) {
                        mCurrentVideoUri = uri;
                        mCurrentVideoUriFromMediaSaved = true;
                        onVideoSaved();
                        //然后通过notifyNewMedia又走到CameraActivity中的notifyNewMedia方法，从上面的log打印也可以看到
                        mActivity.notifyNewMedia(uri);
                    }
                }
            };
private class VideoSaveTask extends AsyncTask <Void, Void, Uri> {
        private String path;
        private final ContentValues values;
        private final OnMediaSavedListener listener;
        private final ContentResolver resolver;

        public VideoSaveTask(String path, ContentValues values, OnMediaSavedListener l,
                             ContentResolver r) {
            this.path = path;
            this.values = new ContentValues(values);
            this.listener = l;
            this.resolver = r;
        }

        @Override
        protected Uri doInBackground(Void... v) {
            Uri uri = null;
            try {
                Uri videoTable = Uri.parse(VIDEO_BASE_URI);
                uri = resolver.insert(videoTable, values);

                // Rename the video file to the final name. This avoids other
                // apps reading incomplete data.  We need to do it after we are
                // certain that the previous insert to MediaProvider is completed.
                String finalName = values.getAsString(Video.Media.DATA);
                finalName = finalName.substring(0, finalName.length() - 4);
                if (path.startsWith("/storage") && !path.startsWith(Storage.FLASH_DIR) && !(new File(Storage.EXTENAL_SD).canWrite())) {
                    path = path.replaceFirst("/storage/" , "/mnt/media_rw/");
                    if (finalName.startsWith("/storage") && !finalName.startsWith(Storage.FLASH_DIR))
                        finalName = finalName.replaceFirst("/storage/" , "/mnt/media_rw/");
                }
                File finalFile = new File(finalName);
                if (new File(path).renameTo(finalFile)) {
                    path = finalName;
                }
                values.put(Video.Media.DATA, finalName);
                resolver.update(uri, values, null, null);
            } catch (Exception e) {
                // We failed to insert into the database. This can happen if
                // the SD card is unmounted.
                Log.e(TAG, "failed to add video to media store", e);
                uri = null;
            } finally {
                Log.v(TAG, "Current video URI: " + uri);
            }
            return uri;
        }

        @Override
        protected void onPostExecute(Uri uri) {
            if (listener != null) {
                listener.onMediaSaved(uri);
            }
        }
    }

• notifyNewMedia方法这里可以看到里面又new了一个AsyncTask，这里直接看onPostExecute方法，上图中log的打印也可以证实这一点

@Override
    public void notifyNewMedia(Uri uri) {
        if (mPaused) {
            return;
        }
        // TODO: This method is running on the main thread. Also we should get
        // rid of that AsyncTask.

        updateStorageSpaceAndHint(null);
        ContentResolver cr = getContentResolver();
        String mimeType = cr.getType(uri);
        Log.v(TAG,"===============notifyNewMedia===================");
        if(mimeType == null) {
            Log.e(TAG, "Can't find video data in content resolver:" + uri);
            return;
        }
        FilmstripItem newData = null;
        if (FilmstripItemUtils.isMimeTypeVideo(mimeType)) {
            sendBroadcast(new Intent(CameraUtil.ACTION_NEW_VIDEO, uri).setFlags(Intent.FLAG_GRANT_READ_URI_PERMISSION));
            newData = mVideoItemFactory.queryContentUri(uri);
            if (newData == null) {
                Log.e(TAG, "Can't find video data in content resolver:" + uri);
                return;
            }
        } else if (FilmstripItemUtils.isMimeTypeImage(mimeType)) {
            CameraUtil.broadcastNewPicture(mAppContext, uri);
            newData = mPhotoItemFactory.queryContentUri(uri);
            if (newData == null) {
                Log.e(TAG, "Can't find photo data in content resolver:" + uri);
                return;
            }
        } else {
            Log.w(TAG, "Unknown new media with MIME type:" + mimeType + ", uri:" + uri);
            return;
        }

        // We are preloading the metadata for new video since we need the
        // rotation info for the thumbnail.
        new AsyncTask<FilmstripItem, Void, FilmstripItem>() {
            @Override
            protected FilmstripItem doInBackground(FilmstripItem... params) {
                FilmstripItem data = params[0];
                MetadataLoader.loadMetadata(getAndroidContext(), data);
                return data;
            }

            @Override
            protected void onPostExecute(final FilmstripItem data) {
                // TODO: Figure out why sometimes the data is aleady there.
                mDataAdapter.addOrUpdate(data);

                // Legacy modules don't use CaptureSession, so we show the capture indicator when
                // the item was safed.
                //不管是拍照还是录像模块都会进入
                if (mCurrentModule instanceof PhotoModule ||
                        mCurrentModule instanceof VideoModule) {
                    AsyncTask.THREAD_POOL_EXECUTOR.execute(new Runnable() {
                        @Override
                        public void run() {
                            Log.d(TAG, "generateThumbnail onPostExecute");
                            //然后在这里获取缩略图，这里主要是通过FilmstripItem的实现类VideoItem
                            final Optional<Bitmap> bitmap = data.generateThumbnail(
                                    mAboveFilmstripControlLayout.getWidth(),
                                    mAboveFilmstripControlLayout.getMeasuredHeight());
                            //有缩略图就显示出来，但是log显示了一个空指针，因此可以判断肯定是获取缩略图有问题，导致无法显示
                            if (bitmap.isPresent()) {
                                indicateCapture(bitmap.get(), 0);
                            }
                            Log.d(TAG, "generateThumbnail onPostExecute end");
                        }
                    });
                }
            }
        }.executeOnExecutor(AsyncTask.THREAD_POOL_EXECUTOR, newData);
    }

VideoItem中generateThumbnail方法
@Override
    public Optional<Bitmap> generateThumbnail(int boundingWidthPx, int boundingHeightPx) {
        return Optional.fromNullable(FilmstripItemUtils.loadVideoThumbnail(
                getData().getFilePath()));
    }

• 然后可以看到真正获取缩略图是通过MediaMetadataRetriever的两个方法获取的，loadVideoThumbnail 和loadVideoThumbnail end也和上面的log相对应

/**
     * Loads the thumbnail of a video.
     *
     * @param path The path to the video file.
     * @return {@code null} if the loading failed.
     */
    public static Bitmap loadVideoThumbnail(String path) {
        Log.d(TAG, "MediaMetadataRetriever loadVideoThumbnail");
        Bitmap bitmap = null;
        MediaMetadataRetriever retriever = new MediaMetadataRetriever();
        try {
            retriever.setDataSource(path);
            //从log可以看出这两个函数都是native函数，并且这两个函数目前获取都为空
            byte[] data = retriever.getEmbeddedPicture();
            if (data != null) {
                bitmap = BitmapFactory.decodeByteArray(data, 0, data.length);
            }
            if (bitmap == null) {
                bitmap = retriever.getFrameAtTime();
            }
        } catch (Exception e) {
            Log.e(TAG, "MediaMetadataRetriever.setDataSource() fail:" + e.getMessage());
        }
        retriever.release();
        Log.d(TAG, "MediaMetadataRetriever loadVideoThumbnail end");
        return bitmap;
    }

• 然后我们找到了3399_Android10\frameworks\base\media\jni\android_media_MediaMetadataRetriever.cpp中的getFrameAtTime方法

static jobject android_media_MediaMetadataRetriever_getFrameAtTime(
        JNIEnv *env, jobject thiz, jlong timeUs, jint option, jint dst_width, jint dst_height)
{
    ALOGV("getFrameAtTime: %lld us option: %d dst width: %d heigh: %d",
            (long long)timeUs, option, dst_width, dst_height);
    sp<MediaMetadataRetriever> retriever = getRetriever(env, thiz);
    if (retriever == 0) {
        jniThrowException(env, "java/lang/IllegalStateException", "No retriever available");
        return NULL;
    }

    // Call native method to retrieve a video frame
    VideoFrame *videoFrame = NULL;
    //可以看到这里调用了retriever->getFrameAtTime方法
    //然后我们需要找到retriever
    sp<IMemory> frameMemory = retriever->getFrameAtTime方法(timeUs, option);
    if (frameMemory != 0) {  // cast the shared structure to a VideoFrame object
        videoFrame = static_cast<VideoFrame *>(frameMemory->pointer());
    }
    if (videoFrame == NULL) {
        //这里的打印也和log一致
        ALOGE("getFrameAtTime: videoFrame is a NULL pointer");
        return NULL;
    }

    return getBitmapFromVideoFrame(env, videoFrame, dst_width, dst_height, kRGB_565_SkColorType);
}

• 最后我们找到了3399_Android10\frameworks\av\media\libmediaplayerservice\MetadataRetrieverClient.cpp文件

static sp<MediaMetadataRetrieverBase> createRetriever(player_type playerType)
{
    sp<MediaMetadataRetrieverBase> p;
    char value[PROPERTY_VALUE_MAX];

    switch (playerType) {
        case STAGEFRIGHT_PLAYER:
        case NU_PLAYER:
        {
            p = new StagefrightMetadataRetriever;
            break;
        }
        //原生的Android10是没有这个case的，这里是RK添加的一个case，我们可以通过设置属性cts_gts.status 为true来进行验证
        //最后发现就是这里有问题，我们可以修改为使用原生的StagefrightMetadataRetriever即修复这个bug
        case ROCKIT_PLAYER:
        {
            if (property_get("cts_gts.status", value, NULL)
                    && !strcasecmp("true", value)) {
                p = new StagefrightMetadataRetriever;
            } else {
				p = new RockitMetadataRetriever;
            }
            break;
        }
        default:
            // TODO:
            // support for TEST_PLAYER
            ALOGE("player type %d is not supported",  playerType);
            break;
    }
    if (p == NULL) {
        ALOGE("failed to create a retriever object");
    }
    return p;
}