Android Jetpack 之CameraX(1.0.0-beta07)使用

CameraX

CameraX 是一个 Jetpack 支持库，目的是为了简化相机应用开发工作。相比打开Camera2动不动就是上百行代码，CameraX使用起来是方便太多了。 本篇介绍Camerax的基本使用方法，以及从如何处理从camera获取到的图像数据（旋转转为nv12）。网上关于CameraX的博客基本上都是从官网翻译的，都是用的kotlin实现的，本篇使用的是java语言。

使用

引入依赖：

	def camerax_version = "1.0.0-beta07"
    // CameraX core library using camera2 implementation
    implementation "androidx.camera:camera-camera2:$camerax_version"
    // CameraX Lifecycle Library
    implementation "androidx.camera:camera-lifecycle:$camerax_version"
    // CameraX View class
    implementation "androidx.camera:camera-view:1.0.0-alpha14"

权限需要动态申请（android.permission.CAMERA）。预览的布局可以使用PreviewView，这是一种可以剪裁、缩放和旋转以确保正确显示的 View。当相机处于活动状态时，图片预览会流式传输到 PreviewView 中的 Surface。它是对SurfaceView和TextureView的封装。

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context=".camerax.CameraxActivity">

    <androidx.camera.view.PreviewView
        android:id="@+id/view_finder"
        android:layout_alignParentTop="true"
        android:layout_alignParentRight="true"
        android:layout_width="match_parent"
        android:layout_height="match_parent"/>

    <ImageButton
        android:id="@+id/take_photo"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true"
        android:layout_centerHorizontal="true"
        android:layout_marginBottom="15dp"
        android:background="@mipmap/take_photo"
        android:onClick="onClick"
        android:text="拍照"/>

    <ImageButton
        android:id="@+id/change_camera"
        android:layout_width="wrap_content"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true"
        android:layout_toLeftOf="@id/take_photo"
        android:layout_marginBottom="15dp"
        android:layout_marginRight="50dp"
        android:background="@mipmap/change"
        android:onClick="onClick" />

</RelativeLayout>

Activity代码，这里封装了个CameraUtil，具体打开camera是在CameraUtil中。另外关于拍照结果的回调是在子线程中的，不能直接更新UI。

import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import androidx.camera.core.ImageCapture;
import androidx.camera.core.ImageCaptureException;
import androidx.camera.view.PreviewView;
import butterknife.BindView;
import butterknife.ButterKnife;

import android.app.Activity;
import android.os.Bundle;
import android.os.Environment;
import android.util.Log;
import android.view.View;
import android.widget.Toast;

import com.example.chat.R;
import com.example.chatlibrary.CameraUtil;

import java.io.File;

public class CameraxActivity extends AppCompatActivity implements ImageCapture.OnImageSavedCallback{

    @BindView(R.id.view_finder)
    PreviewView viewFinder;
    private CameraUtil cameraUtil;


    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_camerax);
        ButterKnife.bind(this);
        init();
    }

    public void onClick(View view) {
        switch (view.getId()) {
            case R.id.take_photo:
                takePhoto();
                break;
            case R.id.change_camera:
                cameraUtil.switchCamera();
                break;
        }
    }

    private void init() {
        cameraUtil = new CameraUtil(viewFinder, this);
        cameraUtil.setImageSavedCallback(this);
        cameraUtil.openCamera();
    }

    private void takePhoto() {
        File rootFile = Environment.getExternalStorageDirectory();
        String path = rootFile + "/DCIM";
        cameraUtil.takePhoto(path);
    }

	/**
	* 图片保存成功的回调
	**/
    @Override
    public void onImageSaved(@NonNull ImageCapture.OutputFileResults outputFileResults) {
        this.runOnUiThread(()->{
            Toast.makeText(this, "take picture success", Toast.LENGTH_SHORT).show();
        });
    }

	/**
	* 图片保存失败的回调
	**/
    @Override
    public void onError(@NonNull ImageCaptureException exception) {
        Log.d("gsy","onError ="+exception);
        this.runOnUiThread(()-> {
            Toast.makeText(this, "onError", Toast.LENGTH_SHORT).show();
        });
    }
}

关键代码是bindToLifecycle，通过调用它就能实现打开camera。

public class CameraUtil {
    private PreviewView viewFinder;
    private Context context;
    Preview preview;
    private int lensFacing;
    private int displayId;
    private Camera camera;
    private ProcessCameraProvider cameraProvider;
    private ExecutorService cameraExecutor;

    private final String FILENAME = "yyyy-MM-dd-HH-mm-ss";
    private final String PHOTO_EXTENSION = ".jpg";
    private final String PHOTO_PREFIX = "/IMG-";

    private double RATIO_4_3_VALUE = 4.0 / 3.0;
    private double RATIO_16_9_VALUE = 16.0 / 9.0;
    private final Long ANIMATION_SLOW_MILLIS = 100L;
    private final Long ANIMATION_FAST_MILLIS = 50L;

    private ImageCapture imageCapture;
    private ViewGroup parent;
    private ImageAnalysis imageAnalysis;
    private ImageAnalysis.Analyzer analyzer;
    private ImageCapture.OnImageSavedCallback imageSavedCallback;
    private int with,height;

    public CameraUtil(PreviewView viewFinder, Context context) {
        this.viewFinder = viewFinder;
        this.context = context;
        cameraExecutor = Executors.newSingleThreadExecutor();
    }

    /**
     * 打开camera
     */
    public void openCamera() {
        viewFinder.post(() -> {
            displayId = viewFinder.getDisplay().getDisplayId();
            updateTransform();
            //设置camera和use cases
            setCamera();
        });
    }

    public void setImageSavedCallback(ImageCapture.OnImageSavedCallback imageSavedCallback) {
        this.imageSavedCallback = imageSavedCallback;
    }

    /**
     * 初始化CameraX, 准备camera use cases
     */
    private void setCamera() {
        //拿到当前进程关联的ProcessCameraProvider
        ListenableFuture<ProcessCameraProvider> instance = ProcessCameraProvider.getInstance(context);
        instance.addListener(() -> {
            try {
                // CameraProvider
                cameraProvider = instance.get();
                lensFacing = CameraSelector.LENS_FACING_BACK;
                bindCameraUseCases();
            } catch (ExecutionException e) {
                e.printStackTrace();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        }, ContextCompat.getMainExecutor(context));
    }

    public int getWith(){
        return with;
    }

    public int getHeight(){
        return height;
    }

    /**
     * 声明并且绑定preview,capture,analysis
     */
    private void bindCameraUseCases() {
        //获取屏幕分辨率用于设置设置camera
        DisplayMetrics displayMetrics = new DisplayMetrics();
        viewFinder.getDisplay().getRealMetrics(displayMetrics);
        int screenAspectRatio = aspectRatio(displayMetrics.widthPixels, displayMetrics.heightPixels);
        with = displayMetrics.widthPixels;
        height = displayMetrics.heightPixels;
        int rotation = viewFinder.getDisplay().getRotation();
        CameraSelector cameraSelector = new CameraSelector.Builder()
                //设置打开前摄还是后摄
                .requireLensFacing(lensFacing)
                .build();
        //预览
        preview = new Preview.Builder()
                //设置预览图像的宽高比
                .setTargetAspectRatio(screenAspectRatio)
                //设置方向
                .setTargetRotation(rotation)
                .build();
        //拍照
        imageCapture = new ImageCapture.Builder()
                .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
                .setTargetAspectRatio(screenAspectRatio)
                .setTargetRotation(rotation)
                .build();

        imageAnalysis = new ImageAnalysis.Builder()
                .setTargetAspectRatio(screenAspectRatio)
                .setTargetRotation(rotation)
                .build();
        if (analyzer == null){
            analyzer = new DefaultAnalyzer();
        }
        imageAnalysis.setAnalyzer(cameraExecutor,analyzer);

        //重新bind前必须先unbind
        cameraProvider.unbindAll();
        //只要可用的use-case都可以添加进来
        camera = cameraProvider.bindToLifecycle((AppCompatActivity) context, cameraSelector, preview, imageCapture,imageAnalysis);
        //将viewFinder的surface provider和preview绑定
        preview.setSurfaceProvider(viewFinder.createSurfaceProvider());
    }

    public void setAnalyzer(ImageAnalysis.Analyzer analyzer){
        this.analyzer = analyzer;
    }

    public void updateTransform() {
        parent = (ViewGroup) viewFinder.getParent();
        //移除之前的viewFinder,重新添加
        parent.removeView(viewFinder);
        parent.addView(viewFinder, 0);
    }

    /**
     * 计算传入的宽高，得到一个最合适的预览尺寸比率
     *
     * @param width
     * @param height
     * @return
     */
    private int aspectRatio(int width, int height) {
        double previewRatio = (double) Math.max(width, height) / Math.min(width, height);
        if (Math.abs(previewRatio - RATIO_4_3_VALUE) <= Math.abs(previewRatio - RATIO_16_9_VALUE)) {
            return AspectRatio.RATIO_4_3;
        }
        return AspectRatio.RATIO_16_9;
    }

    public void takePhoto(String path) {
        File outFile = createFile(path);
        ImageCapture.Metadata metadata = new ImageCapture.Metadata();
        //如果是前摄需要设置左右翻转
        metadata.setReversedHorizontal(lensFacing == CameraSelector.LENS_FACING_FRONT);
        //创建OutputFileOptions对象 包含文件和metadata
        ImageCapture.OutputFileOptions outputOptuins = new ImageCapture.OutputFileOptions.Builder(outFile)
                .setMetadata(metadata)
                .build();
        //调用拍照，此时OnImageSavedCallback是没有运行在主线程的，所以如果要更改ui记得切换线程 imageSavedCallback
        imageCapture.takePicture(outputOptuins, cameraExecutor, imageSavedCallback);
        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
            playAnimation();
        }
    }

    /**
     * 拍照时屏幕闪一下
     */
    private void playAnimation() {
        parent.postDelayed(() -> {
            parent.setForeground(new ColorDrawable(Color.WHITE));
            //记得要设置回来
            parent.postDelayed(() -> {
                parent.setForeground(null);
            }, ANIMATION_FAST_MILLIS);
        }, ANIMATION_SLOW_MILLIS);
    }

    /**
     * 创建jpg文件
     *
     * @return
     */
    private File createFile(String path) {
        SimpleDateFormat simpleDateFormat = new SimpleDateFormat(FILENAME, Locale.CHINA);
        //path+时间.jpg
        File outFile = new File(path + PHOTO_PREFIX + simpleDateFormat.format(System.currentTimeMillis()) + PHOTO_EXTENSION);
        return outFile;
    }

    /**
     * 判断是否有前摄
     *
     * @return
     */
    public boolean hasFrontCamera() {
        try {
            return cameraProvider.hasCamera(CameraSelector.DEFAULT_FRONT_CAMERA);
        } catch (CameraInfoUnavailableException e) {
            e.printStackTrace();
        }
        return false;
    }

    /**
     * 判断是否有后摄
     *
     * @return
     */
    public boolean hasBackCamera() {
        try {
            return cameraProvider.hasCamera(CameraSelector.DEFAULT_BACK_CAMERA);
        } catch (CameraInfoUnavailableException e) {
            e.printStackTrace();
        }
        return false;
    }
	
	//切换前后摄
    public void switchCamera(){
        if(lensFacing == CameraSelector.LENS_FACING_FRONT){
            lensFacing = CameraSelector.LENS_FACING_BACK;
        }else{
            lensFacing = CameraSelector.LENS_FACING_FRONT;
        }
        bindCameraUseCases();
    }

    /**
     * 关闭camera
     */
    public void stop(){
        cameraProvider.unbindAll();
        cameraExecutor.shutdown();
    }

}

预览效果如下。

分析图片需要用到ImageAnalysis，定义一个类实现ImageAnalysis.Analyzer接口。然后调用ImageAnalysis的setAnalyzer方法设置。

图片分析可以分为两种模式：阻塞模式和非阻塞模式。通过使用 STRATEGY_BLOCK_PRODUCER 调用 setBackpressureStrategy() 可以启用阻塞模式。在此模式下，执行器会依序从相应相机接收帧；这意味着，如果 analyze() 方法所用的时间超过单帧在当前帧速率下的延迟时间，所接收的帧便可能不再是最新的帧，因为在该方法返回之前，新帧会被阻止进入流水线。

通过使用 STRATEGY_KEEP_ONLY_LATEST 调用 setBackpressureStrategy() 可以启用非阻塞模式。在此模式下，执行程序在调用 analyze() 方法时会从相机接收最新的可用帧。如果此方法所用的时间超过单帧在当前帧速率下的延迟时间，它可能会跳过某些帧，以便 analyze() 在下一次接收数据时获取相机流水线中的最新可用帧。

从 analyze() 返回前，请通过调用 image.close() 关闭图像引用，以避免阻塞其他图像的生成（导致预览停顿）并避免可能出现的图像丢失。CameraX 会生成 YUV_420_888 格式的图片，如果要进行编码需要将其转为NV12编码格式，而且camerx和之前的camera2一样，图像是横着的，所以还需要进行旋转才能得到正常的图像。

import android.util.Size;
import java.nio.ByteBuffer;
import java.util.concurrent.locks.ReentrantLock;
import androidx.annotation.NonNull;
import androidx.camera.core.ImageAnalysis;
import androidx.camera.core.ImageProxy;
import com.example.chat.util.ImageUtil;

public class CameraAnalyzer implements ImageAnalysis.Analyzer {

    private ReentrantLock lock = new ReentrantLock();
    private byte[] arrayY;
    private byte[] arrayU;
    private byte[] arrayV;
    private Size size;
    private int realWidth;
    private int realHeight;
    private int factor = 3 / 2;

    private byte[] nv21;
    private byte[] nv21_rotated;

    private CameraCallback cameraCallback;
    private boolean isReady = false;

    public CameraAnalyzer() {

    }

    public void setCameraCallback(CameraCallback cameraCallback){
        this.cameraCallback = cameraCallback;
    }

    @Override
    public void analyze(@NonNull ImageProxy image) {
        lock.lock();
        try {
            ImageProxy.PlaneProxy[] planes = image.getPlanes();
            ByteBuffer bufferY = planes[0].getBuffer();
            ByteBuffer bufferU = planes[1].getBuffer();
            ByteBuffer bufferV = planes[2].getBuffer();
            if (arrayY == null) {
                arrayY = new byte[bufferY.limit() - bufferY.position()];
                arrayU = new byte[bufferU.limit() - bufferU.position()];
                arrayV = new byte[bufferV.limit() - bufferV.position()];
                size = new Size(image.getWidth(), image.getHeight());
                realWidth = image.getHeight();
                realHeight = image.getWidth();
            }

            if (bufferY.remaining() == arrayY.length) {
                bufferY.get(arrayY);
                bufferU.get(arrayU);
                bufferV.get(arrayV);
                if (nv21 == null) {
                    nv21 = new byte[realHeight * realWidth * 3 / 2];
                }
                if (nv21_rotated == null) {
                    nv21_rotated = new byte[realHeight * realWidth * 3 / 2];
                }
                if (!isReady && cameraCallback != null) {
                    cameraCallback.cameraReady(size);
                    isReady = true;
                }

                ImageUtil.yuvToNv21(arrayY, arrayU, arrayV, nv21, realWidth, realHeight);
                nv21_rotated = ImageUtil.rotateYUV420Degree90(nv21, image.getWidth(), image.getHeight(), 90);
                byte[] temp = ImageUtil.nv21ToNv12(nv21_rotated);

                if (cameraCallback != null) {
                	//通知编码层进行编码
                    cameraCallback.getCameraFrame(temp);
                }

            }
        } finally {
            lock.unlock();
        }
        image.close();
    }

    /**
     * getCameraFrame  拍出一帧回调
     * cameraReady camera打开回调
     */
    public interface CameraCallback {
        void getCameraFrame(byte[] data);
        void cameraReady(Size size);
    }
}

得到NV21编码数据的代码：

    public static void yuvToNv21(byte[] y, byte[] u, byte[] v, byte[] nv21, int stride, int height) {
        System.arraycopy(y, 0, nv21, 0, y.length);
        // 注意，若length值为 y.length * 3 / 2 会有数组越界的风险，需使用真实数据长度计算
        int length = y.length + u.length / 2 + v.length / 2;
        int uIndex = 0, vIndex = 0;
        for (int i = stride * height; i < length; i += 2) {
            nv21[i] = v[vIndex];
            nv21[i + 1] = u[uIndex];
            vIndex += 2;
            uIndex += 2;
        }
    }

图像旋转90度的代码：

    public static byte[] rotateYUV420Degree90(byte[] input, int width, int height, int rotation) {
        int frameSize = width * height;
        int qFrameSize = frameSize / 4;
        byte[] output = new byte[frameSize + 2 * qFrameSize];


        boolean swap = (rotation == 90 || rotation == 270);
        boolean yflip = (rotation == 90 || rotation == 180);
        boolean xflip = (rotation == 270 || rotation == 180);
        for (int x = 0; x < width; x++) {
            for (int y = 0; y < height; y++) {
                int xo = x, yo = y;
                int w = width, h = height;
                int xi = xo, yi = yo;
                if (swap) {
                    xi = w * yo / h;
                    yi = h * xo / w;
                }
                if (yflip) {
                    yi = h - yi - 1;
                }
                if (xflip) {
                    xi = w - xi - 1;
                }
                output[w * yo + xo] = input[w * yi + xi];
                int fs = w * h;
                int qs = (fs >> 2);
                xi = (xi >> 1);
                yi = (yi >> 1);
                xo = (xo >> 1);
                yo = (yo >> 1);
                w = (w >> 1);
                h = (h >> 1);
// adjust for interleave here
                int ui = fs + (w * yi + xi) * 2;
                int uo = fs + (w * yo + xo) * 2;
// and here
                int vi = ui + 1;
                int vo = uo + 1;
                output[uo] = input[ui];
                output[vo] = input[vi];
            }
        }
        return output;
    }

NV21转成NV12的代码：

    public static byte[] nv21ToNv12(byte[] nv21) {
        byte[] nv12 = new byte[nv21.length];
        int len = nv12.length * 2 / 3;
        System.arraycopy(nv21, 0, nv12, 0, len);
        int index = len;
        while (index < nv12.length - 1) {
            nv12[index] = nv21[index + 1];
            nv12[index + 1] = nv21[index];
            index += 2;
        }
        return nv12;
    }

demo地址：https://github.com/hdguosiyuan/VideoChat

demo实现了预览，拍照功能。同时实现分析图片功能，其实就实现是局域网下的视频通话。一台机器作为服务端，从camera获取图片数据，经过处理后进行编码，然后推流出去，另一台机器作为客户端，同样从camera获取数据推流，两边都需要将收到的视频码流解码然后渲染显示到surface上。本端还需要将本机的camera预览显示到自己窗口上。

效果如图所示，目前音频编码传输还没添加，等有空后会继续完善。