前言
这章主要讲述 怎么调用aar 实现 手势识别,其他mediapipe 工程 也一样
1: 创建 android 工程
testhand
1> 生成 *.binarypb 文件
按照官方 https://google.github.io/mediapipe/getting_started/android_archive_library.html
cd mediapipe 目录(WORKSPACE 所在目录)
bazel build -c opt mediapipe/graphs/hand_tracking:hand_tracking_mobile_gpu_binary_graph
生成 hand_tracking_mobile_gpu.binarypb
把相应文件拷贝到 assets 目录下
除了 aar binarypb 还有那些文件需要具体看 BUILD 的 assets(图中1)
manifest_values 有部分参数值(图中3)
自己生成的MainActivity 参考(图中2)
2:修改代码
MainActivity.java代码如下
package com.example.testhand;
import androidx.annotation.NonNull;
import androidx.appcompat.app.AppCompatActivity;
import android.graphics.SurfaceTexture;
import android.os.Bundle;
/
import android.util.Log;
import android.util.Size;
import android.view.SurfaceHolder;
import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;
import android.widget.Button;
import java.util.Map;
import java.util.HashMap;
import java.util.List;
//
import com.google.mediapipe.components.CameraHelper;
import com.google.mediapipe.components.CameraXPreviewHelper;
import com.google.mediapipe.components.ExternalTextureConverter;
import com.google.mediapipe.components.FrameProcessor;
import com.google.mediapipe.components.PermissionHelper;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmarkList;
import com.google.mediapipe.formats.proto.LandmarkProto.NormalizedLandmark;
import com.google.mediapipe.framework.AndroidAssetUtil;
import com.google.mediapipe.framework.AndroidPacketGetter;
import com.google.mediapipe.framework.Packet;
import com.google.mediapipe.framework.PacketGetter;
import com.google.mediapipe.glutil.EglManager;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.mediapipe.framework.AndroidPacketCreator;
///
public class MainActivity extends AppCompatActivity {
private static final String TAG = "MainActivity";
// 资源文件和流输出名
private static final String BINARY_GRAPH_NAME = "hand_tracking_mobile_gpu.binarypb";
private static final String INPUT_VIDEO_STREAM_NAME = "input_video";
private static final String OUTPUT_VIDEO_STREAM_NAME = "output_video";
private static final String OUTPUT_HAND_PRESENCE_STREAM_NAME = "hand_presence";
private static final String OUTPUT_LANDMARKS_STREAM_NAME = "hand_landmarks";
private static final String INPUT_NUM_HANDS_SIDE_PACKET_NAME = "num_hands";
// Max number of hands to detect/process.
private static final int NUM_HANDS = 2;
private static final String INPUT_MODEL_COMPLEXITY = "model_complexity";
private SurfaceTexture previewFrameTexture;
private SurfaceView previewDisplayView;
private EglManager eglManager;
private FrameProcessor processor;
private ExternalTextureConverter converter;
private CameraXPreviewHelper cameraHelper;
private boolean handPresence;
// 所使用的摄像头
private static final boolean USE_FRONT_CAMERA = true;
// 因为OpenGL表示图像时假设图像原点在左下角,而MediaPipe通常假设图像原点在左上角,所以要翻转
private static final boolean FLIP_FRAMES_VERTICALLY = true;
// 加载动态库
static {
System.loadLibrary("mediapipe_jni");
System.loadLibrary("opencv_java3");
}
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
previewDisplayView = new SurfaceView(this);
setupPreviewDisplayView();
// 获取权限
PermissionHelper.checkAndRequestCameraPermissions(this);
// 初始化assets管理器,以便MediaPipe应用资源
AndroidAssetUtil.initializeNativeAssetManager(this);
eglManager = new EglManager(null);
// 通过加载获取一个帧处理器
processor = new FrameProcessor(this,
eglManager.getNativeContext(),
BINARY_GRAPH_NAME,
INPUT_VIDEO_STREAM_NAME,
OUTPUT_VIDEO_STREAM_NAME);
processor.getVideoSurfaceOutput().setFlipY(FLIP_FRAMES_VERTICALLY);
AndroidPacketCreator packetCreator = processor.getPacketCreator();
Map<String, Packet> inputSidePackets = new HashMap<>();
inputSidePackets.put(INPUT_NUM_HANDS_SIDE_PACKET_NAME, packetCreator.createInt32(NUM_HANDS));
inputSidePackets.put(
INPUT_MODEL_COMPLEXITY, packetCreator.createInt32(0)); //"modelComplexity": "0" # 0=lite, 1=heavy, not specified=heavy
processor.setInputSidePackets(inputSidePackets);
// 获取手的关键点模型输出
processor.addPacketCallback(
OUTPUT_LANDMARKS_STREAM_NAME,
(packet) -> {
Log.v(TAG, "Received multi-hand landmarks packet.");
List<NormalizedLandmarkList> multiHandLandmarks =
PacketGetter.getProtoVector(packet, NormalizedLandmarkList.parser());
Log.v(
TAG,
"[TS:"
+ packet.getTimestamp()
+ "] "
+ getMultiHandLandmarksDebugString(multiHandLandmarks));
});
}
private String getMultiHandLandmarksDebugString(List<NormalizedLandmarkList> multiHandLandmarks) {
if (multiHandLandmarks.isEmpty()) {
return "No hand landmarks";
}
String multiHandLandmarksStr = "Number of hands detected: " + multiHandLandmarks.size() + "\n";
int handIndex = 0;
for (NormalizedLandmarkList landmarks : multiHandLandmarks) {
multiHandLandmarksStr +=
"\t#Hand landmarks for hand[" + handIndex + "]: " + landmarks.getLandmarkCount() + "\n";
int landmarkIndex = 0;
for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
multiHandLandmarksStr +=
"\t\tLandmark ["
+ landmarkIndex
+ "]: ("
+ landmark.getX()
+ ", "
+ landmark.getY()
+ ", "
+ landmark.getZ()
+ ")\n";
++landmarkIndex;
}
++handIndex;
}
return multiHandLandmarksStr;
}
@Override
protected void onResume() {
super.onResume();
converter = new ExternalTextureConverter(eglManager.getContext(),2);
converter.setFlipY(FLIP_FRAMES_VERTICALLY);
converter.setConsumer(processor);
if (PermissionHelper.cameraPermissionsGranted(this)) {
startCamera();
}
}
@Override
protected void onPause() {
super.onPause();
converter.close();
}
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
PermissionHelper.onRequestPermissionsResult(requestCode, permissions, grantResults);
}
// 计算最佳的预览大小
protected Size computeViewSize(int width, int height) {
return new Size(width, height);
}
protected void onPreviewDisplaySurfaceChanged(SurfaceHolder holder, int format, int width, int height) {
// 设置预览大小
Size viewSize = computeViewSize(width, height);
Size displaySize = cameraHelper.computeDisplaySizeFromViewSize(viewSize);
// 根据是否旋转调整预览图像大小
boolean isCameraRotated = cameraHelper.isCameraRotated();
converter.setSurfaceTextureAndAttachToGLContext(
previewFrameTexture,
isCameraRotated ? displaySize.getHeight() : displaySize.getWidth(),
isCameraRotated ? displaySize.getWidth() : displaySize.getHeight());
}
private void setupPreviewDisplayView() {
previewDisplayView.setVisibility(View.GONE);
ViewGroup viewGroup = findViewById(R.id.preview_display_layout);
viewGroup.addView(previewDisplayView);
previewDisplayView
.getHolder()
.addCallback(
new SurfaceHolder.Callback() {
@Override
public void surfaceCreated(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(holder.getSurface());
}
@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
onPreviewDisplaySurfaceChanged(holder, format, width, height);
}
@Override
public void surfaceDestroyed(SurfaceHolder holder) {
processor.getVideoSurfaceOutput().setSurface(null);
}
});
}
// 相机启动后事件
protected void onCameraStarted(SurfaceTexture surfaceTexture) {
// 显示预览
previewFrameTexture = surfaceTexture;
previewDisplayView.setVisibility(View.VISIBLE);
}
// 设置相机大小
protected Size cameraTargetResolution() {
return null;
}
// 启动相机
public void startCamera() {
cameraHelper = new CameraXPreviewHelper();
cameraHelper.setOnCameraStartedListener(this::onCameraStarted);
CameraHelper.CameraFacing cameraFacing =
USE_FRONT_CAMERA ? CameraHelper.CameraFacing.FRONT : CameraHelper.CameraFacing.BACK;
cameraHelper.startCamera(this, cameraFacing, null, cameraTargetResolution());
}
// 解析关键点
private static String getLandmarksDebugString(NormalizedLandmarkList landmarks) {
int landmarkIndex = 0;
StringBuilder landmarksString = new StringBuilder();
for (NormalizedLandmark landmark : landmarks.getLandmarkList()) {
landmarksString.append("\t\tLandmark[").append(landmarkIndex).append("]: (").append(landmark.getX()).append(", ").append(landmark.getY()).append(", ").append(landmark.getZ()).append(")\n");
++landmarkIndex;
}
return landmarksString.toString();
}
}
activity_main.xml 内容如下
<?xml version="1.0" encoding="utf-8"?>
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context=".MainActivity">
<FrameLayout
android:id="@+id/preview_display_layout"
android:layout_width="match_parent"
android:layout_height="match_parent">
<TextView
android:id="@+id/no_camera_access_view"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:gravity="center"
android:text="相机连接失败" />
</FrameLayout>
</androidx.constraintlayout.widget.ConstraintLayout>
AndroidManifest.xml 如下
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.example.testhand">
<!-- For using the camera -->
<uses-permission android:name="android.permission.CAMERA" />
<uses-feature android:name="android.hardware.camera" />
<uses-feature android:name="android.hardware.camera.autofocus" />
<!-- For MediaPipe -->
<uses-feature android:glEsVersion="0x00020000" android:required="true" />
<application
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true"
android:theme="@style/Theme.Testhand">
<activity android:name=".MainActivity">
<intent-filter>
<action android:name="android.intent.action.MAIN" />
<category android:name="android.intent.category.LAUNCHER" />
</intent-filter>
</activity>
</application>
</manifest>
build.grade 如图 红框的是增加的
3:编译OK, APK 大小 41.7 MB (43,755,528 字节)
4:后续 增加几个简单手势的识别
DEMO工程有需要可以上传供参考