Google_FaceDetetor CameraHal 实现基于RK3288平台的实现。
android.media.FaceDetector
安卓SDK提供人脸检测这个类,用法非常的简单,下面是需要分析一下这个功能的实现。
使用方法
首先,大致看一下从应用层调入到HAL层的流程。
packages/apps/Camera/src/com/android/camera/Camera.java
public void startFaceDetection(){ // Try starting Face Detection .... // start face detection only *after* preview has started if (params.getMaxNumDetectedFaces() > 0){ // camera supports face detection, so can start it: mCamera.startFaceDetection(); } }
getMaxNumDetectedFaces在
frameworks/base/core/java/android/hardware/Camera.java
public int getMaxNumDetectedFaces() { return getInt(KEY_MAX_NUM_DETECTED_FACES_HW, 0); }
这里可以看出来:KEY_MAX_NUM_DETECTED_FACES_HW这个值是否大于说明是否支持人脸检测
接着我们看一下startFaceDetection的调用流程
frameworks/base/core/java/android/hardware/Camera.java
public final void startFaceDetection() { if (mFaceDetectionRunning) { throw new RuntimeException("Face detection is already running"); } _startFaceDetection(CAMERA_FACE_DETECTION_HW); mFaceDetectionRunning = true; }
frameworks/base/core/jni/android_hardware_Camera.cpp
//这里start,stop都是通过JNI的方式调用的
static JNINativeMethod camMethods[] = { //省略一些无用函数 { "_startFaceDetection", "(I)V", (void *)android_hardware_Camera_startFaceDetection }, { "_stopFaceDetection", "()V", (void *)android_hardware_Camera_stopFaceDetection}, { "enableFocusMoveCallback", "(I)V", (void *)android_hardware_Camera_enableFocusMoveCallback}, };
static void android_hardware_Camera_startFaceDetection(JNIEnv *env, jobject thiz, jint type) { ALOGV("startFaceDetection"); JNICameraContext* context; sp<Camera> camera = get_native_camera(env, thiz, &context); if (camera == 0) return; status_t rc = camera->sendCommand(CAMERA_CMD_START_FACE_DETECTION, type, 0); if (rc == BAD_VALUE) { char msg[64]; snprintf(msg, sizeof(msg), "invalid face detection type=%d", type); jniThrowException(env, "java/lang/IllegalArgumentException", msg); } else if (rc != NO_ERROR) { jniThrowRuntimeException(env, "start face detection failed"); } }static void android_hardware_Camera_stopFaceDetection(JNIEnv *env, jobject thiz) { ALOGV("stopFaceDetection"); sp<Camera> camera = get_native_camera(env, thiz, NULL); if (camera == 0) return; if (camera->sendCommand(CAMERA_CMD_STOP_FACE_DETECTION, 0, 0) != NO_ERROR) { jniThrowRuntimeException(env, "stop face detection failed"); } }
rk3288 CameraHal 代码实现在:</pre><pre style="white-space:pre-wrap; word-wrap:break-word; font-size:14.3999996185303px; line-height:26px; background-color:rgb(255,255,255)">这里代码就调用的是HAL层实现的函数:
hardware/rk29/camera/CameraHal/下:
我们通过搜索CAMERA_CMD_START_FACE_DETECTION找到相关的实现:
//这里看出去读摄像头支持的数量,同样设置数量的函数我们通过搜索可以发现:int CameraHal::sendCommand(int32_t cmd, int32_t arg1, int32_t arg2) { LOG_FUNCTION_NAME int ret = 0,drv_w,drv_h; Mutex::Autolock lock(mLock); if(cmd == CAMERA_CMD_START_FACE_DETECTION){ const char* szFace = mParameters.get(CameraParameters::KEY_MAX_NUM_DETECTED_FACES_HW); if(!szFace || ( szFace && (strtol(szFace,0,0) == 0))){ ret = BAD_VALUE; return ret; } LOGD("sendCommand start face detection "); enableMsgType(CAMERA_MSG_PREVIEW_METADATA); Message_cam msg; if ((mCommandThread != NULL)) { msg.command = CMD_START_FACE_DETECTION; msg.arg1 = (void*)(NULL); commandThreadCommandQ.put(&msg); } }else if(cmd == CAMERA_CMD_STOP_FACE_DETECTION){ const char* szFace = mParameters.get(CameraParameters::KEY_MAX_NUM_DETECTED_FACES_HW); if(!szFace || ( szFace && (strtol(szFace,0,0) == 0))){ ret = BAD_VALUE; return ret; } //xxxx }
这里发现似乎走了else分支,我们先手动改为1测试。发现情况不容乐观,我们继续看之前的commandThreadCommandQ。struct v4l2_queryctrl facedetect; facedetect.id = V4L2_CID_FACEDETECT; if (!ioctl(mCamFd, VIDIOC_QUERYCTRL, &facedetect)) { params.set(CameraParameters::KEY_MAX_NUM_DETECTED_FACES_HW,"1"); }else{ params.set(CameraParameters::KEY_MAX_NUM_DETECTED_FACES_HW,"0"); }
当然我们找常量比较快:CMD_START_FACE_DETECTION
我们找到这个消息的处理函数:void CameraHal::commandThread() 中队与这个命令的操作就一个调用:
mEventNotifier->startFaceDection(drv_w,drv_h);
走到这里我们先研究一下这个FaceDetetor的实现在Cameral流程:
因为应用的层的使用来,我们很容易得出结论。
我们统统start开启一个线程去处理视频流然后回调某个函数通知上层,最后调用stop 停止这个线程。
接着我们继续往下看。
# grep -rn startFaceDection * AppMsgNotifier.cpp:148:int AppMsgNotifier::startFaceDection(int width,int height)
CameraHal.h:1182: int startFaceDection(int width,int height)
我们先看一下 CameraHal.h 看一下和这个相关的接口
int startFaceDection(int width,int height);//开始人脸检测 void stopFaceDection();//停止人脸检测 bool isNeedSendToFaceDetect(); void notifyNewFaceDecFrame(FramInfo_s* frame); class CameraAppFaceDetThread :public Thread //人脸识别线程 { public: enum FACEDET_THREAD_CMD{ CMD_FACEDET_FACE_DETECT, CMD_FACEDET_PAUSE, CMD_FACEDET_EXIT }; protected: AppMsgNotifier* mAppMsgNotifier; public: CameraAppFaceDetThread(AppMsgNotifier* hw) : Thread(false),mAppMsgNotifier(hw) { } virtual bool threadLoop() { mAppMsgNotifier->faceDetectThread(); return false; } };
</pre>
相关接口十分明显:我们一个个看进去即可。
//这两个函数没什么好解释的,直接进initializeFaceDetec(width, height)看一下int AppMsgNotifier::startFaceDection(int width,int height) { //LOGI("AppMsgNotifier::startFaceDection"); int ret = 0; Mutex::Autolock lock(mFaceDecLock); if(!(mRunningState & STA_RECEIVE_FACEDEC_FRAME)){ if((ret = initializeFaceDetec(width, height)) == 0){ mRunningState |= STA_RECEIVE_FACEDEC_FRAME; mFaceFrameNum = 0; LOG1("start face detection !!"); } mRecMetaDataEn = true; } return ret; } void AppMsgNotifier::stopFaceDection() { Message_cam msg; Semaphore sem; LOG_FUNCTION_NAME { Mutex::Autolock lock(mFaceDecLock); mRecMetaDataEn = false; mRunningState &= ~STA_RECEIVE_FACEDEC_FRAME; } //send msg to stop recording msg.command = CameraAppFaceDetThread::CMD_FACEDET_PAUSE; sem.Create(); msg.arg1 = (void*)(&sem); faceDetThreadCommandQ.put(&msg); if(msg.arg1){ sem.Wait(); } LOG_FUNCTION_NAME_EXIT }
//这里我们要调用libFFTEm.so, 这个东西就是传说中 external/neven下的实现,我们进目录看一下int AppMsgNotifier::initializeFaceDetec(int width,int height){ LOGI("%s(%d): initializeFaceDetec START width =%d,height =%d",__FUNCTION__,__LINE__,width,height); if(!mFaceDetecInit){ //load face detection lib LOGI("%s(%d): LOAD LIBFFETm.so",__FUNCTION__,__LINE__); mFaceDetectorFun.mLibFaceDetectLibHandle = dlopen("libFFTEm.so", RTLD_NOW); if (mFaceDetectorFun.mLibFaceDetectLibHandle == NULL) { LOGE("%s(%d): open libFFTEm.so fail",__FUNCTION__,__LINE__); return -1; } else { mFaceDetectorFun.mFaceDectStartFunc = (FaceDetector_start_func)dlsym(mFaceDetectorFun.mLibFaceDetectLibHandle, "FaceDetector_start"); mFaceDetectorFun.mFaceDectStopFunc = (FaceDetector_stop_func)dlsym(mFaceDetectorFun.mLibFaceDetectLibHandle, "FaceDetector_stop"); mFaceDetectorFun.mFaceDectFindFaceFun = (FaceDetector_findFaces_func)dlsym(mFaceDetectorFun.mLibFaceDetectLibHandle, "FaceDetector_findFaces"); mFaceDetectorFun.mFaceDetector_initizlize_func = (FaceDetector_initizlize_func)dlsym(mFaceDetectorFun.mLibFaceDetectLibHandle, "FaceDetector_initizlize"); mFaceDetectorFun.mFaceDetector_destory_func = (FaceDetector_destory_func)dlsym(mFaceDetectorFun.mLibFaceDetectLibHandle, "FaceDetector_destory");}
# ls Android.mk Embedded FaceDetector_jni.cpp MODULE_LICENSE_APACHE2 CleanSpec.mk FaceDetector.cpp FaceRecEm NOTICE
Android.mk 中我们就看到了确实是由这个工程编译成libFFTEm.so 没错LOCAL_MODULE:= libFFTEm这里我们搜索FaceDetector_start等相关函数并没有找到定义,所以我们判断出来对于人脸检测实际上我们HAL的问题在这里。我们先放开这里不谈,我们知道了start和stop函数,我们并没有看到人脸识别线程相关的内容,我们先找一下这个线程是不是有实现?这里看到线程已经写完了,我们在等一再找一下CMD_FACEDET_FACE_DETECT是谁发送的?先看看processFaceDetectvoid AppMsgNotifier::faceDetectThread(){ while (loop) { case CameraAppFaceDetThread::CMD_FACEDET_FACE_DETECT: if((processFaceDetect(frame) > 0)){ face_detected = true; }else{ face_detected = false; } } }
//这里发现还是回到那几个从so里的处理函数,接着我们看如何出发这个处理的事件:CMD_FACEDET_FACE_DETECTint AppMsgNotifier::processFaceDetect(FramInfo_s* frame) { (*mFaceDetectorFun.mFaceDectFindFaceFun)(mFaceContext,(void*)frame->vir_addr, mCurOrintation,mCurBiasAngle,0, &hasSmileFace, faces, &num);<span style="font-size: 14.4px; white-space: pre-wrap; font-family: Arial, Helvetica, sans-serif;">//其他角度处理的函数</span><span style="font-family: Arial, Helvetica, sans-serif; font-size: 14.4px; white-space: pre-wrap;">}</span>
//所有的线索都断在这个函数notifyNewFaceDecFrame ,我们整理一下目前我们有的思路。void AppMsgNotifier::notifyNewFaceDecFrame(FramInfo_s* frame) { //send to app msg thread Message_cam msg; Mutex::Autolock lock(mFaceDecLock); if((mRecMetaDataEn) && (mRunningState & STA_RECEIVE_FACEDEC_FRAME)) { msg.command = CameraAppFaceDetThread::CMD_FACEDET_FACE_DETECT ; msg.arg2 = (void*)(frame); msg.arg3 = (void*)(frame->used_flag); ///LOGE("AppMsgNotifier::notifyNewFaceDecFrame,send msg"); faceDetThreadCommandQ.put(&msg); }else{ mFrameProvider->returnFrame(frame->frame_index,frame->used_flag); } }
1.我们需要实现
这些函数在CameraHal.h中定义 typedef void (*FaceDetector_start_func)(void *context,int width, int height, int format); typedef void (*FaceDetector_stop_func)(void *context); typedef int (*FaceDetector_findFaces_func)(void *context,void* src, int orientation, float angle, int isDrawRect, int *smileMode,struct RectFace* faces, int* num); typedef void* (*FaceDetector_initizlize_func)(int type, float threshold, int smileMode); typedef void (*FaceDetector_destory_func)(void *context); typedef int (*FaceDector_nofity_func)(struct RectFace* faces, int* num);
struct face_detector_func_s{ void* mLibFaceDetectLibHandle; FaceDetector_start_func mFaceDectStartFunc; FaceDetector_stop_func mFaceDectStopFunc; FaceDetector_findFaces_func mFaceDectFindFaceFun; FaceDetector_initizlize_func mFaceDetector_initizlize_func; FaceDetector_destory_func mFaceDetector_destory_func; };
2.我们要在合适地方调用notifyNewFaceDecFrame
目标明确了之后,这里调用的过程比较简单,我们参考预览界面,因为都是得到一祯图片,最后发现所有的实现和根据我目前的摄像头的情况,
我采取了在这个函数里增加:int CameraUSBAdapter::reprocessFrame(FramInfo_s* frame)//这里通过测试每一帧都会调用这个函数,如果每一帧都进行人脸识别,太消耗性能,最后导致摄像头服务都承受不了图像卡住了,所有我这里每7帧进行一次识别。#if 1 //need to send face detection ? if(mRefEventNotifier->isNeedSendToFaceDetect()){ if( (frameNum++ % BORDER) == 0 ){ mRefEventNotifier->notifyNewFaceDecFrame(frame); } }else{ frameNum=0; } #endif
然后我们看最重要的几个函数的实现,为了方便我直接修改了一下一个函数原型,
参考FaceDetor_jni.cpp实现了一个人脸检测的C封装,这里函数调用其实还是比较容易,看一下实际实现的代码即可。
这里说明一下FaceDetor_jni.cpp是给Java调用的我们只是参考一下人脸检测是怎么调用的,上网搜索居然没有发现相关的参考。因为C++封装会导致so中的符号名改变造成找不到函数,我直接extern "C"用C的方式封装比较简单。
关键代码如下:
extern "C" void * FaceDetector_initizlize(int type, float threshold, int smileMode) { return malloc(sizeof(char)); } extern "C" void FaceDetector_destory(void *context){ if(context!=NULL) free(context); context = NULL; } extern "C" void FaceDetector_start(void *context,int width, int height, int format){ // load the configuration file const char* root = getenv("ANDROID_ROOT"); String8 path(root); //path.appendPath("usr/share/bmd/RFFstd_501.bmd"); path.appendPath("usr/share/bmd/RFFspeed_501.bmd"); const int MAX_FILE_SIZE = 65536; void* initData = malloc( MAX_FILE_SIZE ); /* enough to fit entire file */ int filedesc = open(path.string(), O_RDONLY); int initDataSize = read(filedesc, initData, MAX_FILE_SIZE); close(filedesc); // -------------------------------------------------------------------- btk_SDKCreateParam sdkParam = btk_SDK_defaultParam(); sdkParam.fpMalloc = malloc; sdkParam.fpFree = free; sdkParam.maxImageWidth = width; sdkParam.maxImageHeight = height; btk_Status status = btk_SDK_create(&sdkParam, &g_sdk); // make sure everything went well if (status != btk_STATUS_OK) { return ; } btk_DCRCreateParam dcrParam = btk_DCR_defaultParam(); btk_DCR_create( g_sdk, &dcrParam, &g_dcr ); btk_FaceFinderCreateParam fdParam = btk_FaceFinder_defaultParam(); fdParam.pModuleParam = initData; fdParam.moduleParamSize = initDataSize; fdParam.maxDetectableFaces = g_maxFace; status = btk_FaceFinder_create( g_sdk, &fdParam, &g_fd ); btk_FaceFinder_setRange(g_fd, 20, width/2); /* set eye distance range */ // make sure everything went well if (status != btk_STATUS_OK) { return ; } // free the configuration file free(initData); /* 其他参数初始化 */ g_width = width; g_height = height; } extern "C" int FaceDetector_findFaces(void *context,void* src, int orientation, float angle, int isDrawRect, int *smileMode,struct RectFace* faces, int* num) { // ALOGE("FaceDetector_findFaces...."); // get the fields we need btk_HDCR hdcr = g_dcr; btk_HFaceFinder hfd = g_fd; u32 maxFaces = g_maxFace; u32 width = g_width; u32 height = g_height; int i=0; //ALOGE("#FaceDetector_findFaces....btk_DCR_assignGrayByteImage"); //提取y像素(yuv420 nv12 提取Y) int framesize = width *height ; void *grey = malloc(framesize); memcpy(grey, src, framesize); // run detection,得到人脸数量 btk_DCR_assignGrayByteImage(hdcr, grey, width, height); //ALOGE("#FaceDetector_findFaces....btk_DCR_assignGrayByteImage ok..."); int numberOfFaces = 0; if (btk_FaceFinder_putDCR(hfd, hdcr) == btk_STATUS_OK) { numberOfFaces = btk_FaceFinder_faces(hfd); ALOGE("#FaceDetector_findFaces.... face ==%d",numberOfFaces); } else { ALOGE("ERROR: Return 0 faces because error exists in btk_FaceFinder_putDCR.\n"); *num = -1; free(grey); grey =NULL; return -1; } if(numberOfFaces>5) numberOfFaces=5; if(numberOfFaces >0){ // ALOGE("#face ==..== %d\n",numberOfFaces); getFaceData2(hfd,hdcr, faces,numberOfFaces); } *num =numberOfFaces; free(grey); grey =NULL; //ALOGE("#free..end"); return numberOfFaces; }
//其他的BUG在调试过程中很容易解决了,如果需要C++封装的代码:http://download.csdn.net/download/zmnqazqaz/9880882