在libcamera\hal_common\multiCamera下面定义了所有需要相对于普通Photo模式需要跑额外算法的模式。我们以前摄人像模式为例,跟进下SprdCamera3SinglePortrait的流程。以如下三个关键流程为入口
- configureStream
- processCaptureRequest
- processCaptureResult
configureStream
1,遍历参数 stream_list 中的流数据
int SprdCamera3SinglePortrait::configureStreams(const struct camera3_device *device, camera3_stream_configuration_t *stream_list)
for (size_t i = 0; i < stream_list->num_streams; i++) {
int requestStreamType = getStreamType(stream_list->streams[i]);
HAL_LOGD("i=%d requestStreamType = %d format = %d width=%d height=%d",
i , requestStreamType, stream_list->streams[i]->format, stream_list->streams[i]->width, stream_list->streams[i]->height);
这里有个getStramType的方法,是在其父类SprdCamera3MultiBase中实现的:
int SprdCamera3MultiBase::getStreamType(camera3_stream_t *new_stream) {
int stream_type = 0;
int format = new_stream->format;
if (new_stream->stream_type == CAMERA3_STREAM_OUTPUT) {
if (format == HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED)
format = HAL_PIXEL_FORMAT_YCrCb_420_SP;
switch (format) {
case HAL_PIXEL_FORMAT_YCrCb_420_SP:
if (new_stream->usage & GRALLOC_USAGE_HW_VIDEO_ENCODER) {
stream_type = VIDEO_STREAM;
} else if (new_stream->usage & (uint64_t)BufferUsage::CPU_READ_OFTEN) {
stream_type = CALLBACK_STREAM;
} else {
stream_type = PREVIEW_STREAM;
}
break;
case HAL_PIXEL_FORMAT_RAW16:
if (new_stream->usage & (uint64_t)BufferUsage::CPU_READ_OFTEN) {
stream_type = DEFAULT_STREAM;
}
break;
case HAL_PIXEL_FORMAT_YV12:
case HAL_PIXEL_FORMAT_YCbCr_420_888:
if (new_stream->usage & (uint64_t)BufferUsage::CPU_READ_OFTEN) {
stream_type = DEFAULT_STREAM;
}
break;
case HAL_PIXEL_FORMAT_BLOB:
stream_type = SNAPSHOT_STREAM;
break;
default:
stream_type = DEFAULT_STREAM;
break;
}
} else if (new_stream->stream_type == CAMERA3_STREAM_BIDIRECTIONAL) {
stream_type = CALLBACK_STREAM;
}
return stream_type;
}
getStremType的结果为如下枚举类型
typedef enum {
DEFAULT_STREAM = 1,
PREVIEW_STREAM = 2,
VIDEO_STREAM = 3,
CALLBACK_STREAM = 4,
SNAPSHOT_STREAM = 5,
} streamType_t;
遍历参数流的log:
Line 101316: 05-12 16:41:29.450 470 939 D Cam3SinglePortrait: 3079, configureStreams: i=0 requestStreamType = 2 format = 34 width=960 height=720
Line 101318: 05-12 16:41:29.450 470 939 D Cam3SinglePortrait: 3079, configureStreams: i=1 requestStreamType = 5 format = 33 width=2592 height=1944
参数 camera3_stream_configuration_t 中配了两路流
- 第一个 format为34,即 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED 的预览流
- 第二个 format为33,即 HAL_PIXEL_FORMAT_BLOB 的拍照流
分别对应的streamType为 PREVIEW_STREAM 和 SNAPSHOT_STREAM
2,PREVIEW_STREAM
主要是给 mCaptureThread->mPreviewInitParams 赋值预览的宽、高 数据
if (requestStreamType == PREVIEW_STREAM) {
mPreviewStreamsNum = i;
mCaptureThread->mPreviewInitParams.width =
stream_list->streams[i]->width;
mCaptureThread->mPreviewInitParams.height =
stream_list->streams[i]->height;
if (mCaptureThread->mFirstPreview) {
mCaptureThread->mPreviewWeightParams.sel_x =
mCaptureThread->mPreviewInitParams.width / 2;
mCaptureThread->mPreviewWeightParams.sel_y =
mCaptureThread->mPreviewInitParams.height / 2;
mCaptureThread->mPreviewWeightParams.circle_size =
mCaptureThread->mPreviewInitParams.height *
mCaptureThread->mCircleSizeScale / 100 / 2;
}
}
3,SNAPSHOT_STREAM
3.1,分配 mLocalCapBuffer 的buffer
if (mCaptureWidth != w && mCaptureHeight != h) {
freeLocalCapBuffer();
for (size_t j = 0; j < BLUR_LOCAL_CAPBUFF_NUM; j++) {
if (0 > allocateOne(w, h, &(mLocalCapBuffer[j]), YUV420)) {
HAL_LOGE("request one buf failed.");
continue;
}
}
mCaptureThread->mWeightSize =
(w / mCaptureThread->mCaptureInitParams.Scalingratio) *
(h / mCaptureThread->mCaptureInitParams.Scalingratio) *
sizeof(unsigned short);
mCaptureThread->mOutWeightBuff =
(unsigned short *)malloc(mCaptureThread->mWeightSize);
memset(mCaptureThread->mOutWeightBuff, 0,
mCaptureThread->mWeightSize);
weight_map = (void *)malloc(w * h / 4);
memset(weight_map, 0, w * h / 4);
m_pNearYuvBuffer = (void *)malloc(w * h * 3 / 2);
memset(m_pNearYuvBuffer, 0, w * h * 3 / 2);
}
3.2 给 mCaptureThread->mCaptureInitParams 赋值
w = stream_list->streams[i]->width;
h = stream_list->streams[i]->height;
mCaptureWidth = w;
mCaptureHeight = h;
mCaptureThread->mCaptureInitParams.width = w;
mCaptureThread->mCaptureInitParams.height = h;
3.3 额外创建一路流
以第二路 SNAPSHOT_STREAM 为模板,新建了第三路流,与 SNAPSHOT_STREAM 的不同就是 format 由 HAL_PIXEL_FORMAT_BLOB 改为 HAL_PIXEL_FORMAT_YCbCr_420_888
mCaptureThread->mCaptureStreamsNum = 2;
mCaptureThread->mMainStreams[2].max_buffers = 1;
mCaptureThread->mMainStreams[2].width = w;
mCaptureThread->mMainStreams[2].height = h;
mCaptureThread->mMainStreams[2].format = HAL_PIXEL_FORMAT_YCbCr_420_888;
mCaptureThread->mMainStreams[2].usage = stream_list->streams[i]->usage;
mCaptureThread->mMainStreams[2].stream_type = CAMERA3_STREAM_OUTPUT;
mCaptureThread->mMainStreams[2].data_space = stream_list->streams[i]->data_space;
mCaptureThread->mMainStreams[2].rotation = stream_list->streams[i]->rotation;
pMainStreams[2] = &mCaptureThread->mMainStreams[2];
4, 维护两个stream数组
mCaptureThread->mMainStreams[] 数组有三个stream, 0、1对应configureStream中的预览和拍照流,2 对应上一步中创建的流。
pMainStreams 与 mCaptureThread->mMainStreams[] 的内容是一样的
mCaptureThread->mMainStreams[i] = *stream_list->streams[i];
pMainStreams[i] = &mCaptureThread->mMainStreams[i];
5,初始化各种参数
rc = mCaptureThread->initPortraitParams();
rc = mCaptureThread->initPortraitLightParams();
rc = mCaptureThread->initFaceBeautyParams();
6,构建新的 camera3_stream_configuration 参数,并将其传给SprdCamera3HWI去configure_streams
这个新的 camera3_stream_configuration 是有3路流的,关于SprdCamera3HWI的configure_streams流程,我们在上篇文章中详细讲述了,这里不再重复。我们也可以看出,SprdCamera3SinglePortrait 在这里把流程转到了普通Photo模式的configureStream中。
camera3_stream_configuration mainconfig;
mainconfig = *stream_list;
mainconfig.num_streams = 3;
mainconfig.streams = pMainStreams;
SprdCamera3HWI *hwiMain = m_pPhyCamera[CAM_TYPE_MAIN].hwi;
rc = hwiMain->configure_streams(m_pPhyCamera[CAM_TYPE_MAIN].dev,
&mainconfig);
至此,SprdCamera3SinglePortrait的 configureStream 流程走完了,主要是在原始的参数流中增加了一路。
processCaptureRequest
预览请求
1,遍历参数camera3_capture_request_t 的数据
int SprdCamera3SinglePortrait::processCaptureRequest(
const struct camera3_device *device, camera3_capture_request_t *request)
for (size_t i = 0, j = 0; i < req->num_output_buffers; i++) {
int requestStreamType = getStreamType(request->output_buffers[i].stream);
HAL_LOGD("num_output_buffers:%d, streamtype:%d width:%d height:%d format=%d",
req->num_output_buffers, requestStreamType, request->output_buffers[i].stream->width, request->output_buffers[i].stream->height, request->output_buffers[i].stream->format);
log打印:
processCaptureRequest: num_output_buffers:1, streamtype:4 width:960 height:720 format=34
预览请求只有一个outputBuffer,且format为 HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED。
且获取到的stream_type为4,即 CALLBACK_STREAM。(这里是有点奇怪,可以看上面 getStreamType 的逻辑,如果 new_stream->usage & (uint64_t)BufferUsage::CPU_READ_OFTEN ,就返回 CALLBACK_STREAM,否则返回 PREVIEW_STREAM)
2,保存当前request信息
只保存预览请求的request:getStreamType(newStream) == CALLBACK_STREAM
saveRequest(req);
void SprdCamera3SinglePortrait::saveRequest(
camera3_capture_request_t *request) {
size_t i = 0;
camera3_stream_t *newStream = NULL;
for (i = 0; i < request->num_output_buffers; i++) {
newStream = (request->output_buffers[i]).stream;
newStream->reserved[0] = NULL;
if (getStreamType(newStream) == CALLBACK_STREAM) {
request_saved_single_portrait_t currRequest;
HAL_LOGV("save request %d", request->frame_number);
Mutex::Autolock l(mRequestLock);
currRequest.frame_number = request->frame_number;
currRequest.buffer = request->output_buffers[i].buffer;
currRequest.stream = request->output_buffers[i].stream;
currRequest.input_buffer = request->input_buffer;
mSavedRequestList.push_back(currRequest);
} else if (getStreamType(newStream) == DEFAULT_STREAM) {
mThumbReq.buffer = request->output_buffers[i].buffer;
mThumbReq.preview_stream = request->output_buffers[i].stream;
mThumbReq.input_buffer = request->input_buffer;
mThumbReq.frame_number = request->frame_number;
HAL_LOGD("save thumb request:id %d, w= %d,h=%d",
request->frame_number, (mThumbReq.preview_stream)->width,
(mThumbReq.preview_stream)->height);
}
}
}
3,走 CALLBACK_STREAM == requestStreamType 分支
- 保存当前请求的stream对象
- out_streams_main的stream对像赋值为之前configureStream的预览流那一路的 stream
- updateBlurWeightParams
else if (CALLBACK_STREAM == requestStreamType){
mSavedReqStreams[mPreviewStreamsNum] = req->output_buffers[i].stream;//保存预览starem, mPreviewStreamsNu = 0
out_streams_main[j++].stream = &mCaptureThread->mMainStreams[mPreviewStreamsNum];
mCaptureThread->updateBlurWeightParams(metaSettings, 0);
outbuff_index++;
}
4,构建新的 camera3_capture_request_t 类型的对象
req_main.num_output_buffers = outbuff_index;//1
req_main.output_buffers = out_streams_main;
req_main.settings = metaSettings.release();
5,通过SprdCamera3HWI去process_capture_request
rc = hwiMain->process_capture_request(m_pPhyCamera[CAM_TYPE_MAIN].dev,
&req_main);
预览请求的流程在这里转到了SprdCamera3HWI中,关于SprdCamera3HWI的process_capture_request流程在上篇文章中做了详细介绍,这里就不在重复了。
SprdCamera3SinglePortrait的预览请求的逻辑主要是修改了请求参数 camera3_capture_request_t 中的 output_buffers的stream对象。
拍照请求
1,保存请求参数中的 buffer、请求参数、output_buffers、stream
mCaptureThread->mSavedResultBuff = request->output_buffers[i].buffer;
mjpegSize = ADP_WIDTH(*request->output_buffers[i].buffer);
memcpy(&mCaptureThread->mSavedCapRequest, req, sizeof(camera3_capture_request_t));
memcpy(&mCaptureThread->mSavedCapReqstreambuff, &req->output_buffers[i], sizeof(camera3_stream_buffer_t));
req->output_buffers[i].stream->reserved[0] = NULL;
mSavedReqStreams[mCaptureThread->mCaptureStreamsNum - 1] = req->output_buffers[i].stream;
2,是否有人脸
if (!mFlushing && mCoverValue == 1 &&
(mBlurMode || mCaptureThread->lightPortraitType != 0 ||
mSinglePortrait->mFaceBeautyFlag) &&
((!(mCaptureThread->mVersion == 3 &&
0 != mCaptureThread->mIspInfo.distance_reminder) &&
!((!mCaptureThread->mIsBlurAlways) &&
mCaptureThread->mCaptureWeightParams.total_roi == 0)) ||
mCaptureThread->mGaussEnable)) {
snap_stream_num = 2;
out_streams_main[j].buffer = &mLocalCapBuffer[0].native_handle;
fb_on = 0;
hwiMain->camera_ioctrl(CAMERA_IOCTRL_SET_CAPTURE_FACE_BEAUTIFY,
&fb_on, NULL);
cynr = 0;
hwiMain->camera_ioctrl(CAMERA_IOCTRL_SET_BLUR_CYNR_NO_FACE,
&cynr, NULL);
HAL_LOGD("if snap_stream_num = 2;");//有人脸
} else {
snap_stream_num = 1;
fb_on = 1;
out_streams_main[j].buffer = (req->output_buffers[i]).buffer;
hwiMain->camera_ioctrl(CAMERA_IOCTRL_SET_CAPTURE_FACE_BEAUTIFY,
&fb_on, NULL);
cynr = 1;
hwiMain->camera_ioctrl(CAMERA_IOCTRL_SET_BLUR_CYNR_NO_FACE,
&cynr, NULL);
HAL_LOGD("if snap_stream_num = 1;");//无人脸
}
3,更新 out_streams_main
- 有人脸,使用mMainStreams[2] — format是 HAL_PIXEL_FORMAT_YCbCr_420_888
- 无人脸,使用mMainStreams[1] ----format是 HAL_PIXEL_FORMAT_BLOB
out_streams_main[j++].stream = &mCaptureThread->mMainStreams[snap_stream_num];
SprdCamera3SinglePortrait的拍照请求的逻辑主要是修改了请求参数 camera3_capture_request_t 中的 output_buffers的stream对象。
processCaptureResult
预览回调
void SprdCamera3SinglePortrait::processCaptureResultMain(camera3_capture_result_t *result){}
1,预览回调的信息
可以看到每一个 result->frame_number 会回调两次:
第一次 output_buffer 为 null
第二次 output_buffer 不为null
uint32_t cur_frame_number = result->frame_number;
const camera3_stream_buffer_t *result_buffer = result->output_buffers;
HAL_LOGD("E cur_frame_number = %d format = %d", cur_frame_number, (result_buffer == NULL ? -1 : result_buffer->stream->format));
Line 48: Line 69753: 05-12 20:36:29.968 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 0 format = -1
Line 54: Line 69760: 05-12 20:36:29.970 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 0 format = 34
Line 60: Line 69874: 05-12 20:36:30.015 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 1 format = -1
Line 66: Line 69881: 05-12 20:36:30.016 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 1 format = 34
Line 72: Line 69971: 05-12 20:36:30.066 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 2 format = -1
Line 79: Line 69979: 05-12 20:36:30.068 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 2 format = 34
Line 88: Line 70110: 05-12 20:36:30.117 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 3 format = -1
Line 95: Line 70123: 05-12 20:36:30.119 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 3 format = 34
Line 101: Line 70255: 05-12 20:36:30.168 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 4 format = -1
Line 108: Line 70263: 05-12 20:36:30.169 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 4 format = 34
Line 114: Line 70418: 05-12 20:36:30.218 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 5 format = -1
Line 121: Line 70426: 05-12 20:36:30.220 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 5 format = 34
Line 127: Line 70551: 05-12 20:36:30.269 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 6 format = -1
Line 134: Line 70559: 05-12 20:36:30.270 470 7877 D Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 6 format = 34
2,output_buffer 为 null,调用 mCaptureThread->updateBlurWeightParams(metadata, 1);
output_buffer 为 null 的那一次进入如下 if,一些判断之后会调用 updateBlurWeightParams,它是根据当前Setting中face的信息,来计算更新人脸的位置数据,并保存到 mPreviewWeightParams、mCaptureWeightParams中
if (result_buffer == NULL && result->result != NULL) {
HAL_LOGD("if### result->frame_number=%d mCaptureThread->mSavedCapRequest.frame_number=%d mReqState=%d",
result->frame_number, mCaptureThread->mSavedCapRequest.frame_number, mReqState);
if (result->frame_number ==
mCaptureThread->mSavedCapRequest.frame_number &&
0 != result->frame_number) {
HAL_LOGD("return");
if (mReqState != WAIT_FIRST_YUV_STATE) {
HAL_LOGD("hold yuv picture call back, framenumber:%d",
result->frame_number);
return;
} else {
if (mCaptureThread->mIsBlurAlways &&
mCaptureThread->mVersion == 1) {
mCaptureThread->updateBlurWeightParams(metadata, 1);
}
mCaptureThread->mCallbackOps->process_capture_result(
mCaptureThread->mCallbackOps, result);
return;
}
}
if (mReqState == PREVIEW_REQUEST_STATE) {
int32_t sprd3BlurCapVersion = SINGLE_PORTRAIT_CAP_NOAI;
metadata.update(ANDROID_SPRD_BLUR_CAPVERSION, &sprd3BlurCapVersion,
1);
}
if (mReqState != PREVIEW_REQUEST_STATE) {
HAL_LOGD("mCaptureThread->mIsBlurAlways=%d", mCaptureThread->mIsBlurAlways);
if (mCaptureThread->mIsBlurAlways &&
mCaptureThread->mVersion == 1 &&
mReqState == WAIT_FIRST_YUV_STATE) {
mCaptureThread->updateBlurWeightParams(metadata, 1);
}
if (mCaptureThread->mCaptureWeightParams.roi_type == 2 &&
mCaptureThread->mCaptureWeightParams.valid_roi > 0 &&
(portrait_flag || lightportrait_flag != 0 || facebeauty_flag)) {
int32_t sprd3BlurCapVersion = SINGLE_PORTRAIT_CAP_AI;
metadata.update(ANDROID_SPRD_BLUR_CAPVERSION,
&sprd3BlurCapVersion, 1);
} else {
int32_t sprd3BlurCapVersion = SINGLE_PORTRAIT_CAP_NOAI;
metadata.update(ANDROID_SPRD_BLUR_CAPVERSION,
&sprd3BlurCapVersion, 1);
}
} else {
HAL_LOGD("updateBlurWeightParams");
mCaptureThread->updateBlurWeightParams(metadata, 1);
if (2 == m_nPhyCameras && cur_frame_number > 2) {
SprdCamera3HWI *hwiSub = m_pPhyCamera[CAM_TYPE_AUX].hwi;
SprdCamera3HWI *hwiMain = m_pPhyCamera[CAM_TYPE_MAIN].hwi;
mCoverValue = getCoveredValue(metadata, hwiSub, hwiMain,
m_pPhyCamera[CAM_TYPE_AUX].id);
} else {
char prop[PROPERTY_VALUE_MAX] = {
0,
};
property_get("persist.vendor.cam.blur.cov.val", prop, "1");
if (mCaptureThread->mVersion == 3) {
mCoverValue = atoi(prop);
}
}
}
metadata.update(ANDROID_SPRD_LIGHTPORTRAITTIPS, &mCaptureThread->lpt_return_val, 1);
metadata.update(ANDROID_SPRD_BLUR_COVERED, &mCoverValue, 1);
camera3_capture_result_t new_result = *result;
new_result.result = metadata.release();
mCaptureThread->mCallbackOps->process_capture_result(
mCaptureThread->mCallbackOps, &new_result);
free_camera_metadata(
const_cast<camera_metadata_t *>(new_result.result));
HAL_LOGD("cur_frame_number:%d mCoverValue:%d mReqState:%d",
cur_frame_number, mCoverValue, mReqState);
return;
}
3,output_buffer 不为 null ,遍历在processCaptureRrequest中保存的req
List<request_saved_single_portrait_t>::iterator i = mSavedRequestList.begin();
while (i != mSavedRequestList.end()) {
if (i->frame_number == result->frame_number) {}
}
4,有 portrait_flag 并且 有人脸,跑 prevBlurHandle
if(portrait_flag && (mCaptureThread->mLastFaceNum > 0)){
mCaptureThread->prevBlurHandle(result->output_buffers->buffer, buffer_addr,NULL, NULL, NULL);
}
prevBlurHandle 就是跑人像算法了,该算法的使用大致分为如下3步:
ret = sprd_preview_portrait_adpt(SPRD_PREVIEW_PORTRAIT_DEINIT_CMD,&mPortraitPrevDeinitParams);
ret = sprd_preview_portrait_adpt(SPRD_PREVIEW_PORTRAIT_INIT_CMD,&mPortraitPrevInitParams);
ret = sprd_preview_portrait_adpt(SPRD_PREVIEW_PORTRAIT_GET_MASK_CMD,&mPortraitPrevWeightmapParams);
其中在 SPRD_PREVIEW_PORTRAIT_GET_MASK_CMD 命令的时候会传入如下map之后 buffer_addr 的地址
map(result->output_buffers->buffer, &buffer_addr)
5,有美颜标志 facebeauty_flag,跑 doFaceBeauty
if(facebeauty_flag){
struct facebeauty_param_info Ptr_fb_param_prev;
memset(&Ptr_fb_param_prev, 0, sizeof(Ptr_fb_param_prev));
//for get fbpre_param
SprdCamera3HWI *hwiMain = mSinglePortrait->m_pPhyCamera[CAM_TYPE_MAIN].hwi;
rc = hwiMain->camera_ioctrl(CAMERA_IOCTRL_GET_FB_PREV_PARAM,
&Ptr_fb_param_prev, NULL);
if(rc == 0){
rc = mCaptureThread->doFaceBeauty(NULL, buffer_addr,
mCaptureThread->mPreviewInitParams.width,
mCaptureThread->mPreviewInitParams.height,
0, &mSinglePortrait->fbLevels, lightportrait_flag, &Ptr_fb_param_prev);
}else{
rc = mCaptureThread->doFaceBeauty(NULL, buffer_addr,
mCaptureThread->mPreviewInitParams.width,
mCaptureThread->mPreviewInitParams.height,
0, &mSinglePortrait->fbLevels, lightportrait_flag, NULL);
}
}
美颜算法的使用大致分为如个4个步骤
rc = face_beauty_ctrl(&mSinglePortrait->fb_prev, FB_BEAUTY_CONSTRUCT_FACE_CMD, &mSinglePortrait->beauty_face);
rc = face_beauty_ctrl(&mSinglePortrait->fb_prev, FB_BEAUTY_CONSTRUCT_IMAGE_CMD,&mSinglePortrait->beauty_image);
rc = face_beauty_ctrl(&mSinglePortrait->fb_prev, FB_BEAUTY_CONSTRUCT_LEVEL_CMD, facebeautylevel);
rc = face_beauty_ctrl(&mSinglePortrait->fb_prev, FB_BEAUTY_LPT_PROCESS_CMD, &(lpt_param));
其中在 FB_BEAUTY_CONSTRUCT_IMAGE_CMD 命令的时候会传入如下map之后 buffer_addr 的地址
map(result->output_buffers->buffer, &buffer_addr)
6,有 lightportrait_flag 标志,跑 prevLPT
if(lightportrait_flag != 0){
rc = mCaptureThread->prevLPT(buffer_addr,mCaptureThread->mPreviewInitParams.width,
mCaptureThread->mPreviewInitParams.height, lightportrait_flag);
}
LPT 在项目中未使用
7,跑完上述所有算法后,将数据继续向上callback
memcpy(&newResult, result, sizeof(camera3_capture_result_t));
memcpy(&newOutput_buffers, &result->output_buffers[0], sizeof(camera3_stream_buffer_t));
newOutput_buffers.stream = i->stream;
memcpy(newOutput_buffers.stream, result->output_buffers[0].stream, sizeof(camera3_stream_t));
newResult.output_buffers = &newOutput_buffers;
mCaptureThread->mCallbackOps->process_capture_result(mCaptureThread->mCallbackOps, &newResult);
总的来说,就是在原始流程中的预览数据process_capture_result_main回调后,SprdCamera3SinglePortrait 在对这个数据进行虚化、美颜等操作后,在进一步调用 process_capture_result
拍照回调
与预览不同,拍照请求的回调区分有人脸无人脸
我们在processCaptureRequest中看到,拍照请求会区分有人脸 和 无人脸 两种情况
- 有人脸,snap_stream_num = 2;,stream配的格式是 HAL_PIXEL_FORMAT_YCbCr_420_888
- 无人脸,snap_stream_num = 1;,stream配的格式是 HAL_PIXEL_FORMAT_BLOB
有人脸的情况会回调4次
Line 434: Line 76854: 05-13 18:54:06.078 472 3272 D Cam3SinglePortrait: 3766, processCaptureResultMain: E cur_frame_number = 29 format = -1
Line 637: Line 79488: 05-13 18:54:06.947 472 3275 D Cam3SinglePortrait: 3766, processCaptureResultMain: E cur_frame_number = 29 format = 35
Line 1557: Line 91891: 05-13 18:54:11.624 472 3273 D Cam3SinglePortrait: 3766, processCaptureResultMain: E cur_frame_number = 29 format = -1
Line 1562: Line 91897: 05-13 18:54:11.624 472 3273 D Cam3SinglePortrait: 3766, processCaptureResultMain: E cur_frame_number = 29 format = 33
1,有人脸的第一次回调
第一次 result_buffer == NULL 会进入如下if
注意,在拍照的request中会设置:mReqState = WAIT_FIRST_YUV_STATE;
if (result->frame_number ==
mCaptureThread->mSavedCapRequest.frame_number &&
0 != result->frame_number) {
HAL_LOGD("return");
if (mReqState != WAIT_FIRST_YUV_STATE) {
HAL_LOGD("hold yuv picture call back, framenumber:%d",
result->frame_number);
return;
} else {
if (mCaptureThread->mIsBlurAlways &&
mCaptureThread->mVersion == 1) {
mCaptureThread->updateBlurWeightParams(metadata, 1);
}
mCaptureThread->mCallbackOps->process_capture_result(
mCaptureThread->mCallbackOps, result);
return;
}
}
2,有人脸的第二次回调
第二次 format = 35,即 HAL_PIXEL_FORMAT_YCbCr_420_888,此时 result_buffer != NULL,会进入如下if
- 2.1 发送了 SINGLE_PORTRAIT_MSG_DATA_PROC 的message,并且传入 result->output_buffers->buffer
HAL_LOGD("currStreamType = %d, mReqState = %d", currStreamType, mReqState);
if (mReqState != PREVIEW_REQUEST_STATE &&
currStreamType == DEFAULT_STREAM) {
mSnapshotResultReturn = true;
HAL_LOGD("framenumber:%d, receive yuv:%d", cur_frame_number, mReqState);
// if (mhasCallbackStream && mThumbReq.frame_number) {
// thumbYuvProc(result->output_buffers->buffer);
// mCaptureThread->CallBackSnapResult(CAMERA3_BUFFER_STATUS_OK);
// }
single_portrait_queue_msg_t capture_msg;
capture_msg.msg_type = SINGLE_PORTRAIT_MSG_DATA_PROC;
capture_msg.combo_buff.frame_number = result->frame_number;
capture_msg.combo_buff.buffer = result->output_buffers->buffer;
capture_msg.combo_buff.input_buffer = result->input_buffer;
{
hwiMain->setMultiCallBackYuvMode(false);
Mutex::Autolock l(mCaptureThread->mMergequeueMutex);
mCaptureThread->mCaptureMsgList.push_back(capture_msg);
mCaptureThread->mMergequeueSignal.signal();
}
HAL_LOGI("mVersion %d,mReqState %d", mCaptureThread->mVersion,
mSinglePortrait->mReqState);
}
- 2.2 SINGLE_PORTRAIT_MSG_DATA_PROC 的实现
① map mSinglePortrait->mLocalCapBuffer[1].native_handle
buffer_handle_t *const output_buffer = &mSinglePortrait->mLocalCapBuffer[1].native_handle;
mSinglePortrait->map(output_buffer, &output_buff_addr)
② map capture_msg.combo_buff.buffer
mSinglePortrait->map(capture_msg.combo_buff.buffer, &combo_buff_addr)
③调用 blurProcessVer1
case SINGLE_PORTRAIT_MSG_DATA_PROC: {
void *combo_buff_addr = NULL;
void *output_buff_addr = NULL;
buffer_handle_t *const output_buffer =
&mSinglePortrait->mLocalCapBuffer[1].native_handle;
if (mSinglePortrait->map(capture_msg.combo_buff.buffer,
&combo_buff_addr) != NO_ERROR) {
HAL_LOGE("map combo buffer(%p) failed",
capture_msg.combo_buff.buffer);
return false;
}
if (mSinglePortrait->map(output_buffer, &output_buff_addr) !=
NO_ERROR) {
HAL_LOGE("map output buffer(%p) failed", output_buffer);
return false;
}
HAL_LOGD("mFlushing:%d, frame idx:%d", mSinglePortrait->mFlushing,
capture_msg.combo_buff.frame_number);
if (!mSinglePortrait->mFlushing) {
// blur process
switch (mVersion) {
case 1: {
blurProcessVer1(capture_msg.combo_buff.buffer,
combo_buff_addr, output_buffer,
output_buff_addr,
capture_msg.combo_buff.frame_number);
} break;
default:
HAL_LOGD("blur mVersion %d", mVersion);
HAL_LOGD("blurProcessVer error ");
break;
}
}
if (combo_buff_addr != NULL) {
mSinglePortrait->unmap(capture_msg.combo_buff.buffer);
combo_buff_addr = NULL;
}
if (output_buff_addr != NULL) {
mSinglePortrait->unmap(output_buffer);
output_buff_addr = NULL;
}
// yuv reprocess
if (yuvReprocessCaptureRequest(capture_msg.combo_buff.buffer,
output_buffer) == false) {
HAL_LOGE("yuv reprocess false");
return false;
}
} break;
- blurProcessVer1 的实现
① 跑 SPRD_CAPTURE_PORTRAIT_GET_MASK_INFO_CMD 指令
sprd_capture_portrait_adpt(SPRD_CAPTURE_PORTRAIT_GET_MASK_INFO_CMD,&mPortraitCapGetMaskInfoParams);
②调用 getPortraitMask(output_buff_addr, combo_buff_addr, 0, bokehMask, lptMask);
注意这里的参数,
output_buff_addr 是上一步中map mSinglePortrait->mLocalCapBuffer[1].native_handle
combo_buff_addr 是上一步中map capture_msg.combo_buff.buffer ,即 camera3_capture_result_t 中的 buffer
③getPortraitMask 的实现主要包括下面两个指令
在第二条指令中传入了上一步map的连个地址
int SprdCamera3SinglePortrait::CaptureThread:: getPortraitMask(void *output_buff,
void *input_buf1_addr, int vcmCurValue, void *bokehMask, unsigned char *lptMask){}
rc = sprd_capture_portrait_adpt(SPRD_CAPTURE_PORTRAIT_GET_MASK_INFO_CMD,&mPortraitCapGetMaskInfoParams);
rc = sprd_capture_portrait_adpt(SPRD_CAPTURE_PORTRAIT_GET_MASK_CMD,&mPortraitCapGetMaskParams);
④判断 portrait_flag 标志
if(portrait_flag){
ret = capBlurHandle(combo_buffer, combo_buff_addr,
mSinglePortrait->m_pNearYuvBuffer, output_buffer,
output_buff_addr, bokehMask);
}
⑤ capBlurHandle 的实现
调用如下拍照虚化的命令,并传入 combo_buff_addr 和 output_buffer
int SprdCamera3SinglePortrait::CaptureThread::capBlurHandle(
buffer_handle_t *input1, void *input1_addr, void *input2,
buffer_handle_t *output, void *output_addr, void *mask) {}
et = sprd_capture_portrait_adpt(SPRD_CAPTURE_PORTRAIT_PROCESS_CMD,&mPortraitCapRunParams);
⑥判断美颜标志 facebeauty_flag,跑美颜算法(同预览流程中跑美颜算法)
if(facebeauty_flag){
if(!portrait_flag){
HAL_LOGD("feature support:no portrait+fb");
memcpy(output_buff_addr,combo_buff_addr, ADP_BUFSIZE(*combo_buffer));
} else {
HAL_LOGD("feature support:portrait+fb");
}
struct facebeauty_param_info Ptr_fb_param_cap;
memset(&Ptr_fb_param_cap, 0, sizeof(Ptr_fb_param_cap));
//for get fbcap_param
SprdCamera3HWI *hwiMain = mSinglePortrait->m_pPhyCamera[CAM_TYPE_MAIN].hwi;
ret = hwiMain->camera_ioctrl(CAMERA_IOCTRL_GET_FB_CAP_PARAM, &Ptr_fb_param_cap, NULL);
if (ret == 0){
ret = doFaceBeauty(lptMask, output_buff_addr, mCaptureInitParams.width,
mCaptureInitParams.height, 1, &mSinglePortrait->fbLevels_cap,
lightportrait_flag, &Ptr_fb_param_cap);
}else{
ret = doFaceBeauty(lptMask, output_buff_addr, mCaptureInitParams.width,
mCaptureInitParams.height, 1, &mSinglePortrait->fbLevels_cap,
lightportrait_flag, NULL);
}
}
⑦判断 lightportrait_flag (我们项目一般不支持)
if(lightportrait_flag != 0) {
if(!portrait_flag && (!facebeauty_flag)){
HAL_LOGD("feature support:lpt");
memcpy(output_buff_addr,combo_buff_addr, ADP_BUFSIZE(*combo_buffer));
} else if(portrait_flag) {
HAL_LOGD("feature support:portrait + lpt");
} else {
HAL_LOGD("feature support:facebeauty + lpt");
}
ret = capLPT(output_buff_addr,mCaptureInitParams.width,mCaptureInitParams.height,
lptMask, lightportrait_flag);
}
所以 blurProcessVer1 的逻辑就是将 camera3_capture_result_t 中的 buffer map之后得到 combo_buff_addr,然后对 以
combo_buff_addr 为输入,以 map 的 mSinglePortrait->mLocalCapBuffer[1].native_handle 的结果 output_buff_addr 为输出,跑虚化、美颜算法。
- 调用 yuvReprocessCaptureRequest
bool SprdCamera3SinglePortrait::CaptureThread::yuvReprocessCaptureRequest(
buffer_handle_t *combe_buffer, buffer_handle_t *output_buffer)
①构建 两个 camera3_stream_buffer_t
mSavedCapReqstreambuff是在拍照请求 process_capture_request 中保存的 stream
stream 传的是我们在 configureStreams 中构建保存的 mMainStreams [1] ,其 format 为 33,即 HAL_PIXEL_FORMAT_BLOB
camera3_stream_buffer_t output_stream_buff;
camera3_stream_buffer_t input_stream_buff;
memcpy((void *)&input_stream_buff, &mSavedCapReqstreambuff,
sizeof(camera3_stream_buffer_t));
memcpy((void *)&output_stream_buff, &mSavedCapReqstreambuff,
sizeof(camera3_stream_buffer_t));
output_stream_buff.stream = &mMainStreams[mCaptureStreamsNum - 1];
input_stream_buff.stream = &mMainStreams[mCaptureStreamsNum - 1];
② input_stream_buff 的 buffer 使用 output_buffer
input_stream_buff.buffer = output_buffer;
③构建 camera3_capture_request_t
camera3_capture_request_t request;
request.output_buffers = &output_stream_buff;
request.input_buffer = &input_stream_buff;
④ 使用上述 request 调用 process_capture_request
这个request的 frame_number 就是拍照请求的 frame_number ,
mDevMain->hwi->process_capture_request(mDevMain->dev, &request)
3,有人脸的第三次回调
Cam3SinglePortrait: 3763, processCaptureResultMain: E cur_frame_number = 29 format = -1
这次 result->output_buffers == null,进入第一个 if ,return
4,有人脸的第四次回调
Cam3SinglePortrait: 3766, processCaptureResultMain: E cur_frame_number = 29 format = 33
这次format是 33,即 HAL_PIXEL_FORMAT_BLOB,进入processCaptureResultMain 的如下分支
else if (mReqState != PREVIEW_REQUEST_STATE &&
currStreamType == SNAPSHOT_STREAM) {
HAL_LOGI("jpeg callback framenumber:%d, mReqState:%d",
result->frame_number, mReqState);
uint32_t jpeg_size = 0;
uint8_t *jpeg_addr = NULL;
if (mSinglePortrait->map(result->output_buffers->buffer,
(void **)(&jpeg_addr)) == NO_ERROR) {
jpeg_size = getJpegSize(jpeg_addr,
ADP_WIDTH(*result->output_buffers->buffer));
mSinglePortrait->unmap(result->output_buffers->buffer);
jpeg_addr = NULL;
} else {
HAL_LOGE("map buffer(%p) failed", result->output_buffers->buffer);
}
{
if (mNearJpegSize > 0)
mCaptureThread->saveCaptureBlurParams(
result->output_buffers->buffer, jpeg_size);
}
mCaptureThread->CallSnapBackResult(CAMERA3_BUFFER_STATUS_OK);
if (mCaptureThread->mIsBlurAlways && mCaptureThread->mVersion == 1) {
mCaptureThread->mVersion = 3;
mCaptureThread->mCaptureWeightParams.roi_type = 0;
mCaptureThread->mPreviewWeightParams.roi_type = 0;
}
mReqState = PREVIEW_REQUEST_STATE;
}
主要是调用· CallSnapBackResult 去进一步走 process_capture_result
void SprdCamera3SinglePortrait::CaptureThread::CallSnapBackResult(
camera3_buffer_status_t buffer_status) {
camera3_capture_result_t newResult;
camera3_stream_buffer_t newOutput_buffers;
memset(&newOutput_buffers, 0, sizeof(camera3_stream_buffer_t));
memset(&newResult, 0, sizeof(camera3_capture_result_t));
if (mSinglePortrait->mFlushing) {
buffer_status = CAMERA3_BUFFER_STATUS_ERROR;
}
newOutput_buffers.stream =
mSinglePortrait->mSavedReqStreams[mCaptureStreamsNum - 1];
newOutput_buffers.buffer = mSavedResultBuff;
newOutput_buffers.status = buffer_status;
newOutput_buffers.acquire_fence = -1;
newOutput_buffers.release_fence = -1;
newResult.frame_number = mSavedCapRequest.frame_number;
newResult.output_buffers = &newOutput_buffers;
newResult.input_buffer = NULL;
newResult.result = NULL;
newResult.partial_result = 0;
newResult.num_output_buffers = 1;
HAL_LOGD("buffer_status%d frame_number:%d", buffer_status,
newResult.frame_number);
mCallbackOps->process_capture_result(mCallbackOps, &newResult);
}
5,无人脸
无人脸的时候 拍照帧只会回调两次
Line 5206: Line 120535: 05-13 20:16:21.777 474 11862 D Cam3SinglePortrait: 3768, processCaptureResultMain: E cur_frame_number = 60 format = -1
Line 5496: Line 123618: 05-13 20:16:22.791 474 11863 D Cam3SinglePortrait: 3768, processCaptureResultMain: E cur_frame_number = 60 format = 33
无人脸 第一次
进入第一个if ,走process_capture_result后return
mCaptureThread->mCallbackOps->process_capture_result(mCaptureThread->mCallbackOps, result);
无人脸 第二次
与 有人脸第四次 回调一样,调用mCaptureThread->CallSnapBackResult去进一步 process_capture_result。
无人脸 就不需要走虚化和美颜算法了,就没有 有人脸中第二次回调 的跑虚化和美颜的逻辑了。
至此,我们就以前摄人像模式SprdCamera3SinglePortrait为例,说明了人像模式
configureStream
processCaptureRequest(有人脸、无人脸)
processCaptureResultMain(有人脸、无人脸)
这三大流程了。