远程控制平台三之实现模拟点击

手势捕捉

手势的捕捉是比较简单的，只需设置一个主控端SurfaceView的触摸监听即可，需要注意的是，短按、长按和滑动要区分开，我这里是z在OnTouchListener里简单区分，而且没有进行曲线动作的捕获，有需要的可以在这个监听里去完善。

public class ActionListener implements View.OnTouchListener {
    private int mTouchStartX;
    private int mTouchStartY;
    private int mTouchCurrentX;
    private int mTouchCurrentY;
    private int startX;
    private int startY;
    private int endX;
    private int endY;
    private float screenWidth = ScreenUtils.getScreenWidth();
    private float screenHeight = ScreenUtils.getScreenHeight();

    private long startMillis;
    private boolean longPressPerformed;

    public ActionListener(View screenView) {
        screenView.post(new Runnable() {
            @Override
            public void run() {
                screenWidth = screenView.getWidth() * 1.0f;
                screenHeight = screenView.getHeight() * 1.0f;
            }
        });
    }

    @Override
    public boolean onTouch(View view, MotionEvent event) {
        L.e("触摸位置float: " + event.getX() + ", " + event.getY() + ", action" + event.getAction());
        if (event.getAction() == MotionEvent.ACTION_DOWN) {
            longPressPerformed = false;
            startMillis = System.currentTimeMillis();

            mTouchStartX = (int) event.getX();
            mTouchStartY = (int) event.getY();

            startX = mTouchStartX;
            startY = mTouchStartY;
            endX = startX;
            endY = startY;
            return true;
        } else if (event.getAction() == MotionEvent.ACTION_MOVE) {
            mTouchCurrentX = (int) event.getX();
            mTouchCurrentY = (int) event.getY();
            mTouchStartX = mTouchCurrentX;
            mTouchStartY = mTouchCurrentY;
            endX = mTouchCurrentX;
            endY = mTouchCurrentY;
            boolean longThanLongPress = System.currentTimeMillis()-startMillis > 380;
            if (longThanLongPress && !longPressPerformed) {
                int horizonAbs = Math.abs(endX - startX);
                int verticalAbs = Math.abs(endY - startY);
                if (horizonAbs < 100 && verticalAbs < 100) {
                    longPressPerformed = true;
                    MessageSender.INSTANCE.sendCmd(ActionType.LONG_CLICK, startX/screenWidth, startY/screenHeight, endX/screenWidth, endY/screenHeight, System.currentTimeMillis()-startMillis);
                }
            }
        } else if (event.getAction() == MotionEvent.ACTION_UP) {
            // 已经触发了长按
            if (longPressPerformed) {
                return false;
            }
            if (mTouchCurrentX == 0) {
                L.e("点击");
                MessageSender.INSTANCE.sendCmd(ActionType.CLICK, startX/screenWidth, startY/screenHeight, endX/screenWidth, endY/screenHeight, System.currentTimeMillis()-startMillis);
            } else {
                int horizonAbs = Math.abs(endX - startX);
                int verticalAbs = Math.abs(endY - startY);
                if (horizonAbs > 100 || verticalAbs > 100) {
                    L.e("滑动");
                    MessageSender.INSTANCE.sendCmd(ActionType.SWIPE, startX/screenWidth, startY/screenHeight, endX/screenWidth, endY/screenHeight, System.currentTimeMillis()-startMillis);
                } else {
                    L.e("滑动距离太小，当做点击(" + startX + "," + startY + ") -> (" + endX + ", " + endY + ")");
                    MessageSender.INSTANCE.sendCmd(ActionType.CLICK, startX/screenWidth, startY/screenHeight, endX/screenWidth, endY/screenHeight, System.currentTimeMillis()-startMillis);
                }
            }
        }
        // 为了防止出现只有ACTION_DOWN和ACTION_MOVE，没有ACTION_UP的问题
        return false;
    }
}

手势传递

手势的传递其实和推拉流是一样的，我这里也是使用了JeroMQ的“订阅-发布”模式，但和推拉流方向相反，这里的手势消息是从主控端到受控端。

object MessageSender {
    private val controlExecutorService = Executors.newSingleThreadExecutor()

    @Volatile
    private var controlSocket: ZMQ.Socket? = null

    @Volatile
    private var controlContext: ZContext? = null

    @Volatile
    private var controlCmdUrl = ""

    /**
     * 主控端连接服务器
     */
    fun startControlService() {
        controlCmdUrl = "tcp://${FollowManager.currentFollowBean.p}:${KeyValue.receivePortCmd}"
        controlExecutorService.submit(Runnable {
            if (controlSocket != null) {
                return@Runnable
            }

            try {
                controlContext = ZContext()
                controlSocket = controlContext?.createSocket(SocketType.PUB)
                controlSocket?.enableCommonAuth()
                controlSocket?.enableCommonChat()
                controlSocket?.connect(controlCmdUrl)

                // 点击受控端列表item时，会先发送两个指令过去（让受控端开始推流），先等待几秒再发送，因为建立连接需要时间，太早发送会失败
                Thread.sleep(3500)
                sendCmd(ActionType.CLICK, 0.0f, 0.0f, 0.0f, 1.0f, ActionManager.DURATION_CLICK_START)
            } catch (e: Exception) {
                AppUtils.relaunchAppOnChildThread("主控端命令服务失败")
            }
        })
    }

    fun startControl() {
        GlobalContext.getContext().run {
            val controllerIntent = Intent(this, ControlService::class.java)
            ContextCompat.startForegroundService(this, controllerIntent)
        }
    }

    /**
     * 发送指令给受控端，每一个参数都不能为空
     * 命令格式：收信人ID~动作类型（短击，长按，滑动）~startX~startY~endX~endY~发送时间戳（这里的浮点数是屏幕比例不是像素）
     */
    fun sendCmd(
        touchType: Int,
        startX: Float,
        startY: Float,
        endX: Float,
        endY: Float,
        duration: Long
    ) {
        val deviceId = FollowManager.currentFollowBean.i
        if (TextUtils.isEmpty(deviceId)) {
            return
        }
        controlExecutorService.submit(Runnable {
            try {
                controlSocket?.send("${deviceId}${MESSAGE_SPLIT}${touchType}${MESSAGE_SPLIT}${startX}${MESSAGE_SPLIT}${startY}${MESSAGE_SPLIT}${endX}${MESSAGE_SPLIT}${endY}${MESSAGE_SPLIT}${duration}")
            } catch (e: Exception) {
                L.e(e.message)
            }
        })
    }

    fun sendCmdToExplicitDevice(
        touchType: Int,
        startX: Float,
        startY: Float,
        endX: Float,
        endY: Float,
        duration: Long,
        deviceId: String
    ) {
        if (TextUtils.isEmpty(deviceId)) {
            return
        }
        controlExecutorService.submit(Runnable {
            try {
                controlSocket?.send("${deviceId}${MESSAGE_SPLIT}${touchType}${MESSAGE_SPLIT}${startX}${MESSAGE_SPLIT}${startY}${MESSAGE_SPLIT}${endX}${MESSAGE_SPLIT}${endY}${MESSAGE_SPLIT}${duration}")
            } catch (e: Exception) {
                L.e(e.message)
            }
        })
    }



    fun stopControl() {
        GlobalContext.getContext().run {
            val controllerIntent = Intent(this, ControlService::class.java)
            stopService(controllerIntent)
        }
    }

    fun recycleControlService() {
        try {
            controlSocket?.disconnect(controlCmdUrl)
        } catch (e: Exception) {
            L.e(e.message)
        }

        try {
            controlContext?.destroy()
        } catch (e: Exception) {
            L.e(e.message)
        }

        controlContext = null
        controlSocket = null
    }
}

模拟点击

如何实现一个类似手机版按键精灵的功能（模拟点击）？我们需要用到无障碍功能，即实现一个继承自AccessibilityService的类，注意相关配置。

class ActionService : AccessibilityService() {
    override fun onCreate() {
        super.onCreate()
        startForegroundNotification()
        ActionManager.service = this
    }

    override fun onDestroy() {
        super.onDestroy()
        ActionManager.service = null
    }

    override fun onAccessibilityEvent(event: AccessibilityEvent) {
    }
    override fun onInterrupt() {}

    override fun onKeyEvent(event: KeyEvent?): Boolean {
        return super.onKeyEvent(event)
    }
}

清单文件里声明：

<service
    android:name=".ui.follow.action.ActionService"
    android:enabled="true"
    android:exported="true"
    android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
    android:foregroundServiceType="mediaProjection|dataSync">
    <intent-filter>
        <action android:name="android.accessibilityservice.AccessibilityService" />
    </intent-filter>
    <meta-data
        android:name="android.accessibilityservice"
        android:resource="@xml/accessibility_service_config" />
</service>

accessibility_service_config.xml

<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
    android:accessibilityEventTypes="typeAllMask"
    android:accessibilityFeedbackType="feedbackGeneric"
    android:accessibilityFlags="flagRequestFilterKeyEvents"
    android:canPerformGestures="true"
    android:canRequestFilterKeyEvents="true"
    android:canRequestTouchExplorationMode="true"
    android:canRetrieveWindowContent="true"
    android:description="@string/action_desc"
    android:notificationTimeout="100" />

当受控端收到主控端发过来的命令后，解析并执行：

private fun parseCmd(cmdArray: List<String>) {
    // 没有无障碍权限
    if (!ScreenManager.accessibilityEnabled) {
        return
    }

    // 收信人ID~动作类型（短击，长按，滑动）~startX~startY~endX~endY~动作时长（这里的浮点数是屏幕比例不是像素）
    val touchType = cmdArray[1].toInt()
    val startX = cmdArray[2].toFloat()
    val startY = cmdArray[3].toFloat()
    val endX = cmdArray[4].toFloat()
    val endY = cmdArray[5].toFloat()
    val duration = cmdArray[6].toLong()
    lastCmdMillis = if (touchType == ActionType.CONTROL_LEAVE) {
        0L
    } else {
        ActionManager.wakeupScreenIfNeed()
        System.currentTimeMillis()
    }
    when (touchType) {
        ActionType.CLICK -> {
            ActionManager.click(endX, endY)
        }
        ActionType.LONG_CLICK -> {
            ActionManager.longClick(endX, endY)
        }
        ActionType.SWIPE -> {
            ActionManager.swipe(startX, startY, endX, endY, duration)
        }
        ActionType.DESKTOP -> {
            ActionManager.goDesktop()
        }
        ActionType.BACK -> {
            ActionManager.goBack()
        }
        else -> {
            L.e("未知动作")
        }
    }
}

object ActionManager {
    const val DURATION_CLICK_START = 100L
    private const val DURATION_CLICK = 300L
    private const val DURATION_LONG_CLICK = 2000L
    private val screenWidth = ScreenUtils.getScreenWidth()
    private val screenHeight = ScreenUtils.getScreenHeight()

    @Volatile
    var service : ActionService? = null

    private val xMiddle = screenWidth / 2.0f
    private val yMiddle = screenHeight / 2.0f

    private val xLeft = xMiddle / 2
    private val xRight = xLeft * 3

    private val yTop = yMiddle / 2
    private val yBottom = yTop * 3

    /**
     * 从左往右滑动
     */
    fun swipeLeft2Right() {
        val path = Path().apply {
            //moveTo(xLeft, yMiddle)
            moveTo(50.0f, yMiddle)
            lineTo(xRight, yMiddle)
        }
        dispatchSwipeGesture(path)
    }

    /**
     * 从右往左滑动
     */
    fun swipeRight2Left() {
        val path = Path().apply {
            moveTo(xRight, yMiddle)
            lineTo(xLeft, yMiddle)
        }
        dispatchSwipeGesture(path)
    }

    /**
     * 从上往下滑动
     */
    fun swipeTop2Bottom() {
        val path = Path().apply {
            moveTo(xMiddle, yTop)
            lineTo(xMiddle, yBottom)
        }
        dispatchSwipeGesture(path)
    }

    /**
     * 从下往上滑动
     */
    fun swipeBottom2Top() {
        val path = Path().apply {
            moveTo(xMiddle, yBottom)
            lineTo(xMiddle, yTop)
        }
        dispatchSwipeGesture(path)
    }

    /**
     * 滑动
     */
    fun swipe(startX: Float, startY: Float, endX: Float, endY: Float, duration: Long = DURATION_LONG_CLICK) {
        val realStartX = if (startX < 0) 0.001f else startX
        val realStartY = if (startY < 0) 0.001f else startY
        val realEndX = if (endX < 0) 0.001f else endX
        val realEndY = if (endY < 0) 0.001f else endY
        var startPixelX = realStartX * screenWidth
        var startPixelY = realStartY * screenHeight

        if (startPixelX < 100) {
            startPixelX = 1.0f
        } else if (startPixelX > screenWidth -100) {
            startPixelX = screenWidth * 1.0f - 5
        }

        if (startPixelY < 100) {
            startPixelY = 1.0f
        } else if (startPixelY > screenHeight -100) {
            startPixelY = screenHeight * 1.0f - 5
        }

        val path = Path().apply {
            moveTo(startPixelX, startPixelY)
            lineTo(realEndX * screenWidth, realEndY * screenWidth)
        }
        dispatchSwipeGesture(path, duration)
    }

    /**
     * 点击事件
     * 这里的坐标轴可以通过打开开发者选项里的“指针位置（屏幕叠加层显示当前触摸数据）”来获取，这种通过点击指定像素位置的
     * 实现无法做到兼容所有设备，如果要点击特定按钮，可以通过在service里查找控件并触发点击
     */
    fun click(x: Float, y: Float) {
        L.e("点击事件")

        var startPixelX = x * screenWidth
        if (startPixelX < 100) {
            startPixelX = 1.0f
        } else if (startPixelX > screenWidth -100) {
            startPixelX = screenWidth * 1.0f - 5
        }

        var startPixelY = y * screenHeight
        if (startPixelY < 100) {
            startPixelY = 1.0f
        } else if (startPixelY > screenHeight -100) {
            startPixelY = screenHeight * 1.0f - 5
        }

        GlobalContext.runOnUiThread {
            val clickPath = Path()
            clickPath.moveTo(startPixelX, startPixelY)
            val clickStroke = StrokeDescription(clickPath, DURATION_CLICK_START, DURATION_CLICK)
            val gestureDescription = GestureDescription.Builder().addStroke(clickStroke).build()
            service?.dispatchGesture(gestureDescription, null, null)
        }
    }

    fun longClick(x: Float, y: Float) {
        GlobalContext.runOnUiThread {
            val clickPath = Path()
            clickPath.moveTo(x * screenWidth, y * screenHeight)
            val clickStroke = StrokeDescription(clickPath, DURATION_CLICK_START, DURATION_LONG_CLICK)
            val gestureDescription = GestureDescription.Builder().addStroke(clickStroke).build()
            service?.dispatchGesture(gestureDescription, null, null)
        }
    }


    /**
     * 模拟手势滑动
     */
    private fun dispatchSwipeGesture(path: Path, duration: Long = DURATION_LONG_CLICK) {
        L.e("模拟手势滑动")
        GlobalContext.runOnUiThread {
            val stroke = GestureDescription.StrokeDescription(path, DURATION_CLICK_START, DURATION_CLICK)
            service?.dispatchGesture(
                GestureDescription.Builder().addStroke(stroke).build(), null, null
            )
        }
    }

    /**
     * 回到桌面
     */
    fun goDesktop() {
        GlobalContext.runOnUiThread {
            ActivityUtils.startHomeActivity()
        }
    }

    /**
     * 模拟返回
     */
    fun goBack() {
        GlobalContext.runOnUiThread {
            service?.performGlobalAction(AccessibilityService.GLOBAL_ACTION_BACK)
        }
    }

    /**
     * 如果屏幕没亮则唤起屏幕
     */
    fun wakeupScreenIfNeed() {
        if (!isScreenOn() || MessageReceiver.isControlLeave()) {
            L.e("唤起屏幕")
            AppUtils.wakeupScreen()
        } else {
            L.e("屏幕已亮")
        }
    }

    /**
     * 当前屏幕是否亮了
     */
    private fun isScreenOn(): Boolean {
        val dm = GlobalContext.getContext().getSystemService(AppCompatActivity.DISPLAY_SERVICE) as DisplayManager
        for (display in dm.displays) {
            if (display.state != Display.STATE_ON) {
                return false
            }
        }
        return true
    }

    /**
     * 巧妙唤醒屏幕（先弹出关机对话框，再取消这个对话框）
     */
    private fun wakeScreenSkillfully() {
        GlobalContext.runOnUiThread {
            service?.performGlobalAction(AccessibilityService.GLOBAL_ACTION_POWER_DIALOG)
            click(1.0f, 1.0f)
        }
    }
}

精准点击

以上虽然实现了模拟点击的操作，但留下了一个疑问，主控端点击屏幕某个按钮，如何让受控端也模拟点击相应的位置？因为我们都知道，Android设备碎片化太严重了，各种分辨率、尺寸的屏幕，除非主控端和受控端屏幕参数完全一致，否则位置大概率会有所偏差。刚开始我是想计算当前屏幕点击位置的像素位置，传递到受控端后，转换相应的像素，但效果不佳，后来想到一个简单粗暴的方法：干脆不用传像素了，就传屏幕位置的比例，比如在主控端点击的是A(x,y)位置，那么我就计算A位置的x和y分别占当前设备的比例，假设为x’和y’，把（x’, y’）传到受控端，分别乘以受控端的宽和高，就得到了具体要点击的位置（以上代码可以体现）。这样的做法简单粗暴，但缺点显而易见，就是受控端展示在主控端的画面是完全填充我们的SurfaceView的（类似ImageView的fitXY模式），可能会有不同程度的拉伸，但这并不影响我们精确地控制远程设备。