Android使用OpenCV加载深度学习模型 - MobileSSD

要点：

1. 在JNI中使用Opencv-4.2.0

2. 使用CameraX (1.0.0-beta03）

3. 使用MobileSSD caffe 模型

4. 使用renderscript进行YUV420 -> RGBA

5. easyPermissions权限申请

6. 模型文件会在首次使用时拷贝到packageDir

代码：

https://github.com/riskycheng/DetectX/
其中包含所有必要文件，应该clone下来就可以编译运行：

• MobileSSD 模型文件
• Opencv-4.2 所有依赖库和头文件

Github比较慢的同学，可以在这里下载：CSDN下载地址

结果：

在华为Mate 20 pro (麒麟980) 上大概150ms/inference.

说明

如果对速度比较敏感以及不希望CPU load过高，但opencv其实是不支持android上的GPU，更不用说DSP：

enum Target
{
    DNN_TARGET_CPU,
    DNN_TARGET_OPENCL,
    DNN_TARGET_OPENCL_FP16,
    DNN_TARGET_MYRIAD,
    DNN_TARGET_VULKAN,
    DNN_TARGET_FPGA,
    DNN_TARGET_CUDA,
    DNN_TARGET_CUDA_FP16
};

所以类似Qualcomm的SNPE（必须是骁龙平台），tencent的ncnn 或者 小米的mace应该在部署上都要更具优势。

CameraX

CameraX 是Jetpack中为了解决Camera1和Camera2易用性，但是当前还处于beta阶段,每个阶段APIs变化有点大。这次使用的是截止到目前为止最新的1.0.0-beta03版本。

CameraX支持将Preview/Capture/Analysis绑定到到当前Activity的生命周期，避免以往复杂的配置、生命周期管理操作。最方便的是这种插件形式的开发，新增不同的Model/Task，只需要新增对应的Analyzer，这点实在是很爽。

//获取CameraID
final CameraSelector cameraSelector = new CameraSelector.Builder()
                                        .requireLensFacing(CameraSelector.LENS_FACING_BACK)
                                        .build();

//ListenableFuture继承了Future接口，所以可以支持get获取执行结果,即ProcessCameraProvider
final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(this);

//绑定Preview/Capture/Analysis
cameraProviderFuture.addListener(new Runnable() {
    @Override
    public void run() {
        try {
            //get camera provider
            ProcessCameraProvider cameraProvider = (ProcessCameraProvider) cameraProviderFuture.get();

            //bind to use-cases before rebinding them
            cameraProvider.unbindAll();
            mCamera = cameraProvider.bindToLifecycle(
                            MainActivity.this,
                            cameraSelector,
                            buildAnalyzer(analyzer)
                    );
            } catch (Exception e) {
                Log.e(TAG, "fail to open camera >>> " + e.getMessage());
            }
        }
}, ContextCompat.getMainExecutor(this));      


//ImageAnalysis
 public ImageAnalysis buildAnalyzer(ImageAnalysis.Analyzer analyzer) {
    ImageAnalysis resAnalyzer = new ImageAnalysis.Builder().build();
    Executor imageAnalyzerExecutor = Executors.newSingleThreadExecutor();
    resAnalyzer.setAnalyzer(imageAnalyzerExecutor, analyzer);
    return resAnalyzer;
}

//需要一个实现ImageAnalysis.Analyzer接口的实例来构造ImageAnalysis
public class MobileSSDAnalyzer implements ImageAnalysis.Analyzer {
    @Override
    public void analyze(@NonNull ImageProxy image){
        //imageProxy 封装了android.media.Image的主要方法和成员变量，直接用于处理逻辑
    }
}

最新版本用法参考：
CameraXBasic

OpenCV::DNN

使用方法主要参考官方文档：

• How to run deep networks on Android device
• opencv调用Caffe、TensorFlow、Torch、PyTorch训练好的模型
• 深度学习目标检测MobileNet-SSD

extern "C"
JNIEXPORT void JNICALL
Java_com_fatfish_chengjian_utils_JNIManager_DNN_1execute(JNIEnv *env, jobject clazz,
                                                         jobject inputBitmap) {
    LOGI("entering %s", __FUNCTION__);
    uint32_t *_inputBitmap;
    AndroidBitmapInfo bmapInfo;
    AndroidBitmap_getInfo(env, inputBitmap, &bmapInfo);
    AndroidBitmap_lockPixels(env, inputBitmap, (void **) &_inputBitmap);
    auto *image = (uint8_t *) _inputBitmap;

    //get image info
    int width = bmapInfo.width;
    int height = bmapInfo.height;
    int format = bmapInfo.format;

    Mat inputMat_ = Mat(height, width, CV_8UC4);
    inputMat_.data = image;

    Mat inputMat;
    cvtColor(inputMat_, inputMat, COLOR_BGRA2RGB);

    //因为 mobileSSD inputDim 是300x300x3
    Mat blob = blobFromImage(inputMat, 1.0 / 127.5, Size(IN_WIDTH, IN_HEIGHT),
                             Scalar(MEAN_VAL, MEAN_VAL, MEAN_VAL), false, false);

    //set Input : [1,3,300,300]
    g_DNNet.setInput(blob, "data");
   
    //forward output
    Mat detections = g_DNNet.forward("detection_out");

    //reshape to 100x7 : [0-6] ==> [, class, confidence, topLeft_x, topLeft_y, bottomRight_x, bottomRight_y]
    Mat detectionMat(detections.size[2], detections.size[3], CV_32F, detections.ptr<float>());

    for (int i = 0; i < detectionMat.rows; i++) {

        float confidence = detectionMat.at<float>(i, 2);
        if (confidence > THRESHOLD) {
            size_t objectClass = (size_t) (detectionMat.at<float>(i, 1));
            LOGI("new detections[%d].conf=%f, class=%s", i, confidence,
                 mobileSSDClasses[objectClass]);
            
            //计算实际坐标，这里注意的是输出结果是针对原图
            int topLeft_x = static_cast<int>(detectionMat.at<float>(i, 3) * width);
            int topLeft_y = static_cast<int>(detectionMat.at<float>(i, 4) * height);
            int bottomRight_x = static_cast<int>(detectionMat.at<float>(i, 5) * width);
            int bottomRight_y = static_cast<int>(detectionMat.at<float>(i, 6) * height);
            rectangle(inputMat_,
                      Rect(Point(topLeft_x, topLeft_y), Point(bottomRight_x, bottomRight_y)),
                      Scalar(0, 255, 255), 2);
            putText(inputMat_, mobileSSDClasses[objectClass], Point(topLeft_x, topLeft_y), 1, 1.8,
                    Scalar(255, 0, 0), 2);
        }
    }
    AndroidBitmap_unlockPixels(env, inputBitmap);
    inputMat.release();
    LOGI("exiting %s", __FUNCTION__);
}