向日葵是一款很好用的远程操作软件。
一直很好奇这种软件的基本原理是如何的?
今天带大家通过一个简单的项目来探究一下,并实现一个简单的远程操控软件
原理简析
众所周知,向日葵这种远程控制软件的基本功能如下:
- 1.能够建立起连接,实现两端通讯:
这一点用webSocket
来实现 - 1.能够实时将
被控制端
屏幕画面传给控制端
:
画面传给控制端,简单的实现就是循环将被控制端屏幕截图并发给控制端。
截图功能使用java.awt.Robot
提供的相关方法来实现。 - 2.
被控制端
能够接受到控制端
的操作指令(鼠标移动、键盘输入等),并执行对应的操作
复现指令即实现鼠标移动或者键盘按下等操作,我们同样可以使用java.awt.Robot
提供的相关方法来实现。
实现思路
- 1.
被控制端
服务启动后,开始不断截图发送给连接到它的控制端 -
- 1.1 当一段时间无连接时则停止发送
-
- 1.2 当一段时间未收到控制到操作指令时,则停止发送
-
- 1.3 图片考虑压缩后再发送
- 2.
控制端
与被控制端
建立起连接 -
- 2.1
控制端
通过浏览器访问远程窗口页面与被控制端
建立起webSocket连接
- 2.1
-
- 2.2 当
控制端
接受到返回数据后,页面根据数据将图片展示出来 (实际上应该会不断刷新)
- 2.2 当
-
- 2.3 对整个页面增加事件监听(鼠标移动、键入),并将对应的操作发送给
被控制端
- 2.3 对整个页面增加事件监听(鼠标移动、键入),并将对应的操作发送给
关键代码实现
pom文件导入
本项目基于spring-boot实现,主要依赖为spring-boot-starter-websocket
、thumbnailator
(主要用来实现屏幕截图的压缩)
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.bozinan</groupId>
<artifactId>remoteDesktop</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.6.2</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<java.version>1.8</java.version>
</properties>
<dependencies>
<!-- spring-boot-starter-web start -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- spring-boot-starter-web end -->
<!-- websocket start -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-websocket</artifactId>
</dependency>
<!-- websocket end -->
<!-- thumbnailator start -->
<dependency>
<groupId>net.coobird</groupId>
<artifactId>thumbnailator</artifactId>
<version>0.4.8</version>
</dependency>
<!-- thumbnailator end -->
</dependencies>
<build>
<finalName>${artifactId}</finalName>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
RobotService
RobotService主要利用java.awt.Robot实现被控制端的屏幕截图以及指令回放等功能。
package cn.gzsendi.websocket.service;
import java.awt.AWTException;
import java.awt.Dimension;
import java.awt.Rectangle;
import java.awt.Robot;
import java.awt.Toolkit;
import java.awt.event.InputEvent;
import java.awt.image.BufferedImage;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Map;
import javax.imageio.ImageIO;
import javax.swing.ImageIcon;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.web.socket.BinaryMessage;
import org.springframework.web.socket.WebSocketSession;
import cn.gzsendi.framework.utils.JsonUtil;
import cn.gzsendi.websocket.handler.MyWebSocketHandler;
/**
* 定时抓取截图以及处理服务端的事件回放(键盘与鼠标)
*/
@Service
public class RobotService {
private Logger logger = LoggerFactory.getLogger(RobotService.class);
// 记录最后一后键盘或鼠标事件的到达时间
private Long lastestActionTime = System.currentTimeMillis();
// 远程服务端的屏幕宽
private int remoteImageWidth ;
// 远程服务端的屏幕高
private int remoteImageHeigth;
private Robot robot = null;
private Rectangle rectangle = null;
public RobotService() {
try {
// 核心机器人类,用于截图,键盘或鼠标事件的重放执行。
robot = new Robot();
Toolkit toolkit = Toolkit.getDefaultToolkit();
// 获取到远程桌面的屏幕大小信息
Dimension dimension = toolkit.getScreenSize();
rectangle = new Rectangle(0, 0, (int)dimension.getWidth(), (int)dimension.getHeight());
} catch (AWTException e) {
logger.error("",e);
}
}
/**
* 进行截图任务的处理,如果有客户端连接上来,将进行截图并广播发送给所有的客户端
*/
public void startCaputureTask(){
while(true){
try {
//100毫秒检查一次,如果有客户端,并且满足需要截图的条件,就截图一张发给所有的客户端,可以调整这个值,值越小延迟越小
Thread.sleep(1L);
// 遍历所有在线的客户端
Map<String,WebSocketSession> webSocketSessions = MyWebSocketHandler.webSocketSessions;
// 没有websocket客户端连上的话,直接就退出本轮循环,不需要进行截图处理
if(webSocketSessions.size() == 0 ) {
// 避免长时间未连接,第一次进入后无图片展示的问题
lastestActionTime = System.currentTimeMillis();
continue;
}
// 如果超过5秒没有收到键盘或鼠标事件,说明可以停止截图给客户端,节省性能。
if((System.currentTimeMillis() - lastestActionTime) > 5000){
logger.info("exceed 5 seconds not keyboard event arrived, stop send images.");
continue;
}
// 截图
byte[] data = getCapture(robot,rectangle);
ImageIcon icon = new ImageIcon(data);
remoteImageWidth = icon.getIconWidth();
remoteImageHeigth = icon.getIconHeight();
//遍历发送给所有的客户端连接
for(WebSocketSession webSocketSession : webSocketSessions.values()) {
if(webSocketSession.isOpen()) {
webSocketSession.sendMessage(new BinaryMessage(data));
}
}
} catch (Exception e) {
logger.error("startCaputureTaskError",e);
}
}
}
/**
* 得到屏幕截图数据
* @return
*/
private byte[] getCapture(Robot robot,Rectangle rectangle) {
BufferedImage bufferedImage = robot.createScreenCapture(rectangle);
//获得一个内存输出流
ByteArrayOutputStream baos = new ByteArrayOutputStream();
//将图片数据写入内存流中
try {
logger.info("getCapture");
//原始图片,现在用下面的压缩图片法替换了
ImageIO.write(bufferedImage, "jpg", baos);
//进行图片压缩,图片尺寸不变,压缩图片文件大小outputQuality实现,参数1为最高质量
//Thumbnails.of(bufferedImage).scale(1f).outputQuality(0.25f).outputFormat("jpg").toOutputStream(baos);
} catch (IOException e) {
logger.error("图片写入出现异常",e);
}
return baos.toByteArray();
}
//回放处理客户端发送过来的键盘或鼠标事件
public void actionEvent(Map<String,Object> playload){
String openType = JsonUtil.getString(playload, "openType");
if("mousedown".equals(openType)){
//鼠标按下事件
logger.info("鼠标按下事件,{}",JsonUtil.toJSONString(playload));
int clientX = JsonUtil.getInteger(playload, "clientX");
int clientY = JsonUtil.getInteger(playload, "clientY");
int button = JsonUtil.getInteger(playload, "button");
int imageWidth = JsonUtil.getInteger(playload, "imageWidth");
int imageHeight = JsonUtil.getInteger(playload, "imageHeight");
//这里为什么要这样转?说明如下:
//假如浏览器的image区域为1200*800,远程桌面的截图区为900*700
//那么在浏览器上点击了clientX=77,clientY=88这个坐标时,实际上在远程
//桌面上正确的坐标应该为:
//remoteClientX = clientX * remoteImageWidth/imageWidth;
//即:remoteClientX = 77 * 900 / 1200
//remoteClientY同理.
int remoteClientX = clientX * remoteImageWidth/imageWidth;
int remoteClientY = clientY * remoteImageHeigth/imageHeight;
//移动鼠标到正确的坐标
robot.mouseMove( remoteClientX , remoteClientY );
//然后进行鼠标的按下
if(button == 0) {
robot.mousePress(InputEvent.BUTTON1_MASK);//左键
}else if(button == 1) {
robot.mousePress(InputEvent.BUTTON2_MASK);//中间键
}else if(button == 2) {
robot.mousePress(InputEvent.BUTTON3_MASK);//右键
}
}else if("mouseup".equals(openType)){
//鼠标弹开事件
logger.info("鼠标弹开事件,{}",JsonUtil.toJSONString(playload));
int clientX = JsonUtil.getInteger(playload, "clientX");
int clientY = JsonUtil.getInteger(playload, "clientY");
int button = JsonUtil.getInteger(playload, "button");
int imageWidth = JsonUtil.getInteger(playload, "imageWidth");
int imageHeight = JsonUtil.getInteger(playload, "imageHeight");
int remoteClientX = clientX*remoteImageWidth/imageWidth;
int remoteClientY = clientY*remoteImageHeigth/imageHeight;
//移动鼠标到正确的坐标
robot.mouseMove( remoteClientX , remoteClientY );
//然后进行鼠标的弹起
if(button == 0) {
robot.mouseRelease(InputEvent.BUTTON1_MASK);//左键
}else if(button == 1) {
robot.mouseRelease(InputEvent.BUTTON2_MASK);//中间键
}else if(button == 2) {
robot.mouseRelease(InputEvent.BUTTON3_MASK);//右键
}
}else if("mousemove".equals(openType)){
//鼠标移动事件
int clientX = JsonUtil.getInteger(playload, "pageX");
int clientY = JsonUtil.getInteger(playload, "pageY");
int imageWidth = JsonUtil.getInteger(playload, "imageWidth");
int imageHeight = JsonUtil.getInteger(playload, "imageHeight");
int remoteClientX = clientX*remoteImageWidth/imageWidth;
int remoteClientY = clientY*remoteImageHeigth/imageHeight;
//将鼠标进行移动
robot.mouseMove( remoteClientX , remoteClientY );
}else if("keydown".equals(openType)){
//键盘按下事件
logger.info("键盘按下事件,{}",JsonUtil.toJSONString(playload));
int keyCode = JsonUtil.getInteger(playload, "keyCode");
robot.keyPress(changeKeyCode(keyCode));
}else if("keyup".equals(openType)){
//键盘弹开事件
logger.info("键盘弹开事件,{}",JsonUtil.toJSONString(playload));
int keyCode = JsonUtil.getInteger(playload, "keyCode");
robot.keyRelease(changeKeyCode(keyCode));
}
}
//进行keyCode的改变,因为浏览器的键盘事件和Java的awt的事件代码,有些是不一样的,需要进行转换,
//比如浏览器中13表示回车,但在Java的awt中是用10表示
//这里可能转换不全,比如F1-F12键都没有处理,因为浏览器现在没有禁用这些键,如果需要支持,可以继续在这里加上
private int changeKeyCode(int sourceKeyCode){
//回车
if(sourceKeyCode == 13) return 10;
//,< 188 -> 44
if(sourceKeyCode == 188) return 44;
//.>在Js中为190,但在Java中为46
if(sourceKeyCode == 190) return 46;
// /?在Js中为191,但在Java中为47
if(sourceKeyCode == 191) return 47;
//;: 186 -> 59
if(sourceKeyCode == 186) return 59;
//[{ 219 -> 91
if(sourceKeyCode == 219) return 91;
//\| 220 -> 92
if(sourceKeyCode == 220) return 92;
//-_ 189->45
if(sourceKeyCode == 189) return 45;
//=+ 187->61
if(sourceKeyCode == 187) return 61;
//]} 221 -> 93
if(sourceKeyCode == 221) return 93;
//DEL
if(sourceKeyCode == 46) return 127;
//Ins
if(sourceKeyCode == 45) return 155;
return sourceKeyCode;
}
public int getRemoteImageWidth() {
return remoteImageWidth;
}
public void setRemoteImageWidth(int remoteImageWidth) {
this.remoteImageWidth = remoteImageWidth;
}
public int getRemoteImageHeigth() {
return remoteImageHeigth;
}
public void setRemoteImageHeigth(int remoteImageHeigth) {
this.remoteImageHeigth = remoteImageHeigth;
}
public Long getLastestActionTime() {
return lastestActionTime;
}
public void setLastestActionTime(Long lastestActionTime) {
this.lastestActionTime = lastestActionTime;
}
}
MyWebSocketHandler
MyWebSocketHandler类实现WebSocketHandler接口,主要功能是维护一个与控制端的连接,并提供接收到控制端返回指令后的处理逻辑(调用RobotService 中的相关方法进行事件回放)
package cn.gzsendi.websocket.handler;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.socket.BinaryMessage;
import org.springframework.web.socket.CloseStatus;
import org.springframework.web.socket.PongMessage;
import org.springframework.web.socket.TextMessage;
import org.springframework.web.socket.WebSocketHandler;
import org.springframework.web.socket.WebSocketMessage;
import org.springframework.web.socket.WebSocketSession;
import cn.gzsendi.framework.utils.JsonUtil;
import cn.gzsendi.websocket.constant.ConstantPool;
import cn.gzsendi.websocket.service.RobotService;
/**
* @Description: WebSocket处理器
*/
@Component
public class MyWebSocketHandler implements WebSocketHandler{
private Logger logger = LoggerFactory.getLogger(MyWebSocketHandler.class);
public static Map<String,WebSocketSession> webSocketSessions = new ConcurrentHashMap<String, WebSocketSession>();
@Autowired RobotService robotService;
/**
* @Description: 用户连接上WebSocket的回调
* @Param: [webSocketSession]
* @return: void
*/
@Override
public void afterConnectionEstablished(WebSocketSession webSocketSession) throws Exception {
logger.info("用户:{},连接WebSSH", webSocketSession.getAttributes().get(ConstantPool.USER_UUID_KEY));
webSocketSessions.put(webSocketSession.getAttributes().get(ConstantPool.USER_UUID_KEY).toString(), webSocketSession);
}
/**
* @Description: 收到消息的回调
* @Param: [webSocketSession, webSocketMessage]
* @return: void
*/
@Override
public void handleMessage(WebSocketSession webSocketSession, WebSocketMessage<?> webSocketMessage) throws Exception {
//设置更新最后一后键盘或鼠标事件的到达时间
robotService.setLastestActionTime(System.currentTimeMillis());
if (webSocketMessage instanceof TextMessage) {
//logger.info("用户:{},发送命令:{}", webSocketSession.getAttributes().get(ConstantPool.USER_UUID_KEY), webSocketMessage.toString());
Map<String,Object> playload = JsonUtil.castToObject(webSocketMessage.getPayload().toString());
//回放处理客户端发送过来的键盘或鼠标事件,在服务端这边重新执行一遍
robotService.actionEvent(playload);
} else if (webSocketMessage instanceof BinaryMessage) {
} else if (webSocketMessage instanceof PongMessage) {
} else {
logger.error("Unexpected WebSocket message type: " + webSocketMessage);
}
}
/**
* @Description: 出现错误的回调
* @Param: [webSocketSession, throwable]
* @return: void
*/
@Override
public void handleTransportError(WebSocketSession webSocketSession, Throwable throwable) throws Exception {
logger.error("数据传输错误");
}
/**
* @Description: 连接关闭的回调
* @Param: [webSocketSession, closeStatus]
* @return: void
*/
@Override
public void afterConnectionClosed(WebSocketSession webSocketSession, CloseStatus closeStatus) throws Exception {
logger.info("用户:{}断开webssh连接", String.valueOf(webSocketSession.getAttributes().get(ConstantPool.USER_UUID_KEY)));
webSocketSessions.remove(webSocketSession.getAttributes().get(ConstantPool.USER_UUID_KEY).toString());
}
@Override
public boolean supportsPartialMessages() {
return false;
}
}
控制界面实现
控制端界面主要实现与被控制端建立起socket链接,并对返回数据进行重现、监听相关操作并发送给被控制端
<!doctype html>
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>WEB远程桌面</title>
<link rel="stylesheet" href="../css/init.css" />
</head>
<body>
<!--image标签,用于远程桌面的截图显示-->
<img id="imageId" src='' style='position:fixed;width:100%;height:100%' draggable="false"/>
<script src="../js/jquery-3.4.1.min.js"></script>
<script src="../js/websocketclient.js" charset="utf-8"></script>
<script>
//websocketClient
var client = null;
//获取accessToken,简单的请求下后台接口判断accessToken是否正确
//accessToken默认为123456
//如果token有问题,不进行连接
var accessToken = getQueryVariable("accessToken") ;
//请求Token信息验证
$.ajax({
url: window.location.origin + '/tokenController/check?accessToken=' + accessToken,
type: 'get',
success: function (res) {
if(res === "success"){
//启动远程桌面
startRemoteWin();
}else {
alert("accessToken check error.");
}
},
error: function (result) {
}
});
//计算imageId的宽和高变量
var imageWidth = $("#imageId").width();
var imageHeight = $("#imageId").height();
//当浏览器大小变化时,更新imageId的宽和高变量
$(window).resize( function () {
imageWidth = $("#imageId").width();
imageHeight = $("#imageId").height();
});
//Jquery禁用网页右键菜单
$(document).bind("contextmenu",function(e){
return false;
});
//键盘被按下去事件
$(document).keydown(function (event) {
var obj = new Object();
obj.openType = "keydown";
obj.keyCode = event.which || event.keyCode;
client.sendClientData(obj);//将数据通过websocket发送到后台进行重放
//禁用一些快捷键
if (event.ctrlKey && window.event.keyCode==65){ //禁用ctrl + a 功能
return false;
}
//禁用一些快捷键
if (event.ctrlKey && window.event.keyCode==67){ //禁用ctrl + c 功能
return false;
}
//禁用一些快捷键
if (event.ctrlKey && window.event.keyCode==83){ //禁用ctrl + s 功能
return false;
}
//禁用一些快捷键
if (event.ctrlKey && window.event.keyCode==86){ //禁用ctrl + v 功能
return false;
}
//你想禁用其他快捷键时
//console.log(event);
//比如说我按下 A键 keyCode=65 获取到keyCode,然后按以上的方法禁止
//目前F1到F12还没有禁用,如果需要的话可以加上。
});
//键盘被弹起来事件
$(document).keyup(function (event) {
var obj = new Object();
obj.openType = "keyup";
obj.keyCode = event.which || event.keyCode;
client.sendClientData(obj);
});
//鼠标按钮被按下
$(document).mousedown(function (event) {
var obj = new Object();
obj.openType = "mousedown";
obj.button = event.button;
obj.clientX = event.clientX; //需要在后台重新计算转换成远程桌面上的真实的坐标
obj.clientY = event.clientY; //需要在后台重新计算转换成远程桌面上的真实的坐标
obj.imageWidth = imageWidth;
obj.imageHeight = imageHeight;
client.sendClientData(obj);
});
//鼠标按钮被松开
$(document).mouseup(function (event) {
var obj = new Object();
obj.openType = "mouseup";
obj.button = event.button;
obj.clientX = event.clientX;
obj.clientY = event.clientY;
obj.imageWidth = imageWidth;//当前浏览器下image标签占用的宽和高,传这两个值到后台用于修正真实的点击的x和y坐标
obj.imageHeight = imageHeight;
client.sendClientData(obj);
});
//鼠标移动事件
// $(document).mousemove(function(event){
// var obj = new Object();
// obj.openType = "mousemove";
// obj.button = event.button;
// obj.pageX = event.pageX;
// obj.pageY = event.pageY;
// obj.imageWidth = imageWidth;
// obj.imageHeight = imageHeight;
// client.sendClientData(obj);
//
// });
//远程桌面连接函数
function startRemoteWin(options){
//修改title
$('title').html("WEB远程桌面【连接中...】");
client = new WebsocketClient();
//执行连接操作
client.connect({
onError: function (error) {
//连接失败回调
console.log("Error");
//设置连接失败后的title
$('title').html("WEB远程桌面【连接失败】");
},
onConnect: function () {
//连接成功回调
console.log("连接成功回调\r\n");
//设置成功连接后的title
$('title').html("WEB远程桌面【连接成功】");
},
onClose: function () {
//连接关闭回调
console.log("\rconnection closed, now reconnect comtempt..");
//alert("Websocket连接已关闭");
startRemoteWin();
},
onData: function (data) {
//收到数据时回调
//console.log(data);
//判断websocket的消息是二进制还是字符串
if (typeof(data) === 'string') {
console.log("string");
} else {
//console.log("bin");
//后台是通过Java的Awt工具将图片转成了二进制流回来
//因此在这里将二进制流作一下处理,将新传回来的截图imageId的image标签中修改src,达到远程控制的效果
//将图片刷到浏览器上显示
const blob = new Blob([data], { type: "image/jpg" });
document.getElementById("imageId").src = window.URL.createObjectURL(blob);
}
}
});
}
//获取浏览器地址上的url参数
function getQueryVariable(variable){
var query = window.location.search.substring(1);
var vars = query.split("&");
for (var i=0;i<vars.length;i++) {
var pair = vars[i].split("=");
if(pair[0] == variable){return pair[1];}
}
return "";
}
</script>
</body>
</html>
效果展示
可以通过浏览器进行远程控制
源码下载
本项目源码已上传至CSDN资源 spring-boot+webSocket实现向日葵远程操作功能
如有需要请前往下载!
如果本文有帮助到你,请点赞收藏!!!
您的支持是我更新的最大动力!!!!