说明
实际场景中, 客户端可以随时连接随意断开, 但服务端一旦断线, 可能导致客户端不可用, 有必要在服务端不可用时, 切换可用服务端进行重连。服务端的断线可能的情况很多, 此处只考虑2种情况: 非正常断线(如断电、整个服务被kill掉), 正常断线(手动停止netty服务)。非正常断线其实是可以处理正常断线的, 故引入额外的服务, 就像有开发了, 为啥还是需要测试。如果把netty看成一个即时通讯模块的话, 其实, 额外的服务可以是业务模块, 或者其他高可用的模块均可
1. 创建新项目monitor, 放在160机器上
对于维护此服务的高可用, 应该是很简单的, 且服务中的一些配置, 后续可以引入nacos, 来解决发版问题,
1.1 pom.xml信息
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.4.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.hahashou</groupId>
<artifactId>monitor</artifactId>
<version>1.0-SNAPSHOT</version>
<name>monitor</name>
<description>Monitor project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.58</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<excludes>
<exclude>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</exclude>
</excludes>
</configuration>
</plugin>
</plugins>
</build>
</project>
1.2 application.yml信息
server:
port: 31000
spring:
redis:
host: 192.168.109.160
port: 6379
password: root
logging:
level:
com.hahashou.monitor: info
netty:
monitor:
# 当某机器需要升级时, 将服务端id配置在此处, 待客户端连接数不多时, 可直接停机
exclude: netty-server-X
1.3 logback-spring.xml信息
<?xml version="1.0" encoding="utf-8" ?>
<!-- 日志级别(低->高) : TRACE < DEBUG < INFO < WARN < ERROR < FATAL -->
<configuration scan="true" scanPeriod="60 seconds" debug="false">
<!-- 彩色日志依赖的渲染类 -->
<conversionRule conversionWord="clr" converterClass="org.springframework.boot.logging.logback.ColorConverter" />
<conversionRule conversionWord="wex" converterClass="org.springframework.boot.logging.logback.WhitespaceThrowableProxyConverter" />
<conversionRule conversionWord="wEx" converterClass="org.springframework.boot.logging.logback.ExtendedWhitespaceThrowableProxyConverter" />
<!-- 基本属性配置, 更换项目时修改此处 -->
<property name="rootPath" value="monitor-logs"/>
<property name="applicationName" value="monitor"/>
<contextName>${applicationName}</contextName>
<!-- 控制台日志: 彩色渲染输出 -->
<property name="CONSOLE_LOG_PATTERN" value="${CONSOLE_LOG_PATTERN:-%clr(%d{${LOG_DATEFORMAT_PATTERN:-yyyy-MM-dd HH:mm:ss.SSS}}){faint} %clr(${LOG_LEVEL_PATTERN:-%5p}) %clr(${PID:- }){magenta} %clr(---){faint} %clr([%15.15t]){faint} %clr(%-40.40logger{39}){cyan} %clr(:){faint} %m%n${LOG_EXCEPTION_CONVERSION_WORD:-%wEx}}"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>${CONSOLE_LOG_PATTERN}</pattern>
</encoder>
</appender>
<!-- Info日志 -->
<appender name="INFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${rootPath}/${applicationName}-info.log</file>
<encoder>
<!--格式化输出:%d表示日期,%thread表示线程名,%-5level:级别从左显示5个字符宽度%msg:日志消息,%n是换行符-->
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50}-%msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<FileNamePattern>${rootPath}/${applicationName}-info.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<maxFileSize>50MB</maxFileSize>
<maxHistory>30</maxHistory>
</rollingPolicy>
<filter class="ch.qos.logback.classic.filter.LevelFilter">
<level>INFO</level>
</filter>
</appender>
<!-- Error日志 -->
<appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${rootPath}/${applicationName}-error.log</file>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{50} - %msg%n</pattern>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<FileNamePattern>${rootPath}/${applicationName}-error.%d{yyyy-MM-dd}.%i.log</FileNamePattern>
<maxFileSize>50MB</maxFileSize>
<maxHistory>30</maxHistory>
</rollingPolicy>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>ERROR</level>
</filter>
</appender>
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="INFO"/>
<appender-ref ref="ERROR"/>
</root>
<!-- 异步打印 -->
<appender name="FILE_ASYNC" class="ch.qos.logback.classic.AsyncAppender">
<appender-ref ref="STDOUT"/>
<appender-ref ref="INFO"/>
<appender-ref ref="ERROR"/>
</appender>
</configuration>
1.4 新建config包, 新增 RedisConfig类
package com.hahashou.monitor.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
/**
* @description: Redis配置
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Configuration
public class RedisConfig {
@Bean
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) {
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
// 使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值
redisTemplate.setValueSerializer(new GenericJackson2JsonRedisSerializer());
redisTemplate.setHashValueSerializer(new GenericJackson2JsonRedisSerializer());
// 使用StringRedisSerializer来序列化和反序列化redis的key
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.setHashKeySerializer(new StringRedisSerializer());
// 开启事务:redisTemplate.setEnableTransactionSupport(true); 我觉得一般用不到(该操作是为了执行一组命令而设置的)
redisTemplate.setConnectionFactory(redisConnectionFactory);
return redisTemplate;
}
@Bean
public ValueOperations<String, Object> redisOperation(RedisTemplate<String, Object> redisTemplate) {
return redisTemplate.opsForValue();
}
public static String NETTY_SERVER_SET = "NETTY_SERVER_SET";
public static String key(String hostName, String port) {
return hostName + "@" + port;
}
}
1.5 新增service包, 新增 NettyServerService接口
package com.hahashou.monitor.service;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
public interface NettyServerService {
/**
* 在线的服务端
* @return
*/
String nettyServer();
}
1.6 service包下新增impl包, 新增 类
package com.hahashou.monitor.service.impl;
import com.alibaba.fastjson.JSON;
import com.hahashou.monitor.config.RedisConfig;
import com.hahashou.monitor.service.NettyServerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.net.*;
import java.util.*;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Service
@Slf4j
public class NettyServerServiceImpl implements NettyServerService {
@Value("${netty.monitor.exclude}")
private String exclude;
@Resource
private ValueOperations<String, Object> redisOperation;
@Override
public String nettyServer() {
Object nettyServerSet = redisOperation.get(RedisConfig.NETTY_SERVER_SET);
if (nettyServerSet != null) {
Set<String> excludeSet = new HashSet<>(Arrays.asList(exclude.split(",")));
Set<String> nodeList = new HashSet<>(JSON.parseArray(nettyServerSet.toString(), String.class));
// 只做轮询策略, 其他策略想做的自行补充
TreeMap<Long, String> map = new TreeMap<>();
Map<String, Long> nettyLoadMap = new HashMap<>(16);
Set<String> errorSet = new HashSet<>();
for (String hostAndPort : nodeList) {
String hostName = hostAndPort.split("@")[0];
if (excludeSet.contains(hostName)) {
continue;
}
boolean nettyHealth = nettyHealth(hostName);
if (!nettyHealth) {
// 非正常掉线后, 移除在线服务以及删除服务的连接数
errorSet.add(hostAndPort);
redisTemplate.delete(hostAndPort);
continue;
}
Object object = redisOperation.get(hostAndPort);
long connectNumber = object != null ? Long.parseLong(object.toString()) : 0L;
nettyLoadMap.put(hostAndPort, connectNumber);
map.put(connectNumber, hostAndPort);
}
nodeList.removeAll(errorSet);
redisOperation.set(RedisConfig.NETTY_SERVER_SET, JSON.toJSONString(nodeList));
log.info("netty集群负载情况: {}", nettyLoadMap.toString());
String[] split = map.firstEntry().getValue().split("@");
InetAddress byName;
try {
byName = InetAddress.getByName(split[0]);
} catch (UnknownHostException exception) {
log.error("请检查hosts文件配置: {}", exception.getMessage());
return "";
}
return RedisConfig.key(byName.getHostAddress(), split[1]);
}
return "";
}
/**
* 服务端的健康状态
* @param hostName
* @return
*/
public boolean nettyHealth(String hostName) {
boolean health = false;
try {
InetAddress byName = InetAddress.getByName(hostName);
URL url = new URL("http://" + byName.getHostAddress() + ":32000/server/health");
HttpURLConnection httpConnection = (HttpURLConnection) url.openConnection();
httpConnection.setRequestMethod("GET");
httpConnection.setConnectTimeout(1500);
int responseCode = httpConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
health = true;
}
} catch (IOException exception) {
log.error("获取Netty服务端异常: {}", exception.getMessage());
}
return health;
}
}
1.7 新建controller包, 新增 NettyServerController类
package com.hahashou.monitor.controller;
import com.hahashou.monitor.service.NettyServerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@RestController
@RequestMapping("/nettyServer")
@Slf4j
public class NettyServerController {
@Resource
private NettyServerService nettyServerService;
@GetMapping
public String nettyServer() {
return nettyServerService.nettyServer();
}
}
1.8 打包成jar, 放到160服务器 /usr/local/src, 之后编写启动/停止脚本
start-monitor.sh
cd /usr/local/src
java -Dfile.encoding=UTF-8 -jar monitor-1.0-SNAPSHOT.jar &
stop-monitor.sh
ps -ef | grep monitor-1.0-SNAPSHOT.jar | grep -v grep | awk '{print $2}' | xargs kill -9
1.9 修改hosts, 此处应该和服务端的hosts保持一致
vi /etc/hosts
追加内容
192.168.109.161 netty-server-1
192.168.109.162 netty-server-2
192.168.109.163 netty-server-3
1.10 启动服务
sh start-monitor.sh
1.11 开放端口31000端口
2. 服务端改造
Redis key值修改&增加服务端的连接数记录&增加健康检查接口
2.1 修改 RedisConfig类
package com.hahashou.netty.server.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
/**
* @description: Redis配置
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Configuration
public class RedisConfig {
@Bean
public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory redisConnectionFactory) {
RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
// 使用Jackson2JsonRedisSerializer来序列化和反序列化redis的value值
redisTemplate.setValueSerializer(new GenericJackson2JsonRedisSerializer());
redisTemplate.setHashValueSerializer(new GenericJackson2JsonRedisSerializer());
// 使用StringRedisSerializer来序列化和反序列化redis的key
redisTemplate.setKeySerializer(new StringRedisSerializer());
redisTemplate.setHashKeySerializer(new StringRedisSerializer());
// 开启事务:redisTemplate.setEnableTransactionSupport(true); 我觉得一般用不到(该操作是为了执行一组命令而设置的)
redisTemplate.setConnectionFactory(redisConnectionFactory);
return redisTemplate;
}
@Bean
public ValueOperations<String, Object> redisOperation(RedisTemplate<String, Object> redisTemplate) {
return redisTemplate.opsForValue();
}
public static String NETTY_SERVER_LOCK = "NETTY_SERVER_LOCK";
public static String NETTY_SERVER_SET = "NETTY_SERVER_SET";
public static String OFFLINE_MESSAGE = "OFFLINE_MESSAGE_";
public static String key(String hostName, String port) {
return hostName + "@" + port;
}
}
2.2 修改 NettyClient类
package com.hahashou.netty.server.config;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.*;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.concurrent.EventExecutorGroup;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import javax.annotation.PreDestroy;
import java.net.*;
import java.nio.charset.Charset;
/**
* @description: Netty-客户端TCP服务
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Slf4j
public class NettyClient {
@Getter
@Setter
private NioEventLoopGroup clientWorkerGroup;
@Getter
@Setter
private EventExecutorGroup clientBusinessGroup;
public void createClient(NettyClientHandler nettyClientHandler) {
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(clientWorkerGroup)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer<NioSocketChannel>() {
@Override
protected void initChannel(NioSocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new StringDecoder(Charset.forName("UTF-8")));
pipeline.addLast(new StringEncoder(Charset.forName("UTF-8")));
pipeline.addLast(clientBusinessGroup, nettyClientHandler);
}});
try {
InetAddress inetAddress = InetAddress.getByName(nettyClientHandler.getHostName());
SocketAddress socketAddress = new InetSocketAddress(inetAddress, Integer.parseInt(nettyClientHandler.getPort()));
bootstrap.connect(socketAddress).sync().channel();
} catch (UnknownHostException exception) {
log.error("请检查hosts文件是否配置正确 : {}", exception.getMessage());
} catch (InterruptedException exception) {
log.error("客户端中断异常 : {}", exception.getMessage());
}
}
@PreDestroy
public void destroy() {
clientWorkerGroup.shutdownGracefully().syncUninterruptibly();
log.info("客户端关闭成功");
}
}
2.3 修改 NettyClientHandler类
package com.hahashou.netty.server.config;
import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Slf4j
public class NettyClientHandler extends ChannelInboundHandlerAdapter {
@Getter
@Setter
private String userCode;
@Getter
@Setter
private String hostName;
@Getter
@Setter
private String port;
@Resource
private ValueOperations<String, Object> redisOperation;
@Override
public void channelActive(ChannelHandlerContext ctx) {
log.info("{}, 作为客户端, 与其他服务端连接", LocalDateTime.now());
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
NettyStatic.CHANNEL.remove(ctx.channel().id().asLongText());
NettyClientHandler nettyClientHandler = NettyStatic.NETTY_CLIENT_HANDLER
.remove(RedisConfig.key(hostName, port));
NettyClient nettyClient = NettyStatic.NETTY_CLIENT.remove(nettyClientHandler);
nettyClient = null;
nettyClientHandler = null;
System.gc();
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
if (msg != null) {
Message message = JSON.parseObject(msg.toString(), Message.class);
String channelId = message.getChannelId(),
text = message.getText();
if (StringUtils.hasText(channelId)) {
Channel channel = ctx.channel();
message.setUserCode(userCode);
NettyStatic.USER_CHANNEL.put(hostName, channelId);
NettyStatic.CHANNEL.put(channelId, channel);
channel.writeAndFlush(Message.transfer(message));
} else if (StringUtils.hasText(text)) {
String friendUserCode = message.getFriendUserCode();
if (StringUtils.hasText(message.getServerId())) {
String queryChannelId = NettyStatic.USER_CHANNEL.get(friendUserCode);
if (StringUtils.hasText(queryChannelId)) {
Channel channel = NettyStatic.CHANNEL.get(queryChannelId);
if (channel == null) {
offlineMessage(friendUserCode, message);
return;
}
// 此时, 已不需要serverId
message.setServerId(null);
channel.writeAndFlush(Message.transfer(message));
} else {
offlineMessage(friendUserCode, message);
}
}
}
}
}
/**
* 离线消息存储Redis
* @param friendUserCode
* @param message
*/
public void offlineMessage(String friendUserCode, Message message) {
List<Message> messageList = new ArrayList<>();
Object offlineMessage = redisOperation.get(RedisConfig.OFFLINE_MESSAGE + friendUserCode);
if (offlineMessage != null) {
messageList = JSON.parseArray(offlineMessage.toString(), Message.class);
}
messageList.add(message);
redisOperation.set(RedisConfig.OFFLINE_MESSAGE + friendUserCode, JSON.toJSONString(messageList));
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
}
}
2.4 修改 NettyServer类
package com.hahashou.netty.server.config;
import com.alibaba.fastjson.JSON;
import io.netty.bootstrap.ServerBootstrap;
import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.HashedWheelTimer;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
import io.netty.util.concurrent.EventExecutorGroup;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import javax.annotation.PreDestroy;
import javax.annotation.Resource;
import java.nio.charset.Charset;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* @description: Netty-服务端TCP服务
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Component
@Slf4j
public class NettyServer implements TimerTask {
@Value("${netty.server.id}")
private String serverId;
@Value("${netty.server.port}")
private int port;
@Resource
private NioEventLoopGroup bossGroup;
@Resource
private NioEventLoopGroup workerGroup;
@Resource
private EventExecutorGroup businessGroup;
@Resource
private NettyServerHandler nettyServerHandler;
@Resource
private NioEventLoopGroup clientWorkerGroup;
@Resource
private EventExecutorGroup clientBusinessGroup;
@Resource
private RedisTemplate<String, Object> redisTemplate;
@Resource
private ValueOperations<String, Object> redisOperation;
@Resource
private HashedWheelTimer hashedWheelTimer;
@Override
public void run(Timeout timeout) {
Object nettyServerLock = redisOperation.get(RedisConfig.NETTY_SERVER_LOCK);
if (nettyServerLock != null) {
hashedWheelTimer.newTimeout(this, 10, TimeUnit.SECONDS);
return;
}
try {
redisOperation.set(RedisConfig.NETTY_SERVER_LOCK, true);
//String hostAddress = InetAddress.getLocalHost().getHostAddress();
ServerBootstrap serverBootstrap = new ServerBootstrap();
ChannelFuture channelFuture = serverBootstrap.group(bossGroup, workerGroup)
.channel(NioServerSocketChannel.class)
.childHandler(new ChannelInitializer<SocketChannel>() {
@Override
public void initChannel(SocketChannel ch) {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new StringDecoder(Charset.forName("UTF-8")));
pipeline.addLast(new StringEncoder(Charset.forName("UTF-8")));
pipeline.addLast(businessGroup, nettyServerHandler);
}
})
// 服务端可连接队列数,对应TCP/IP协议listen函数中backlog参数
.option(ChannelOption.SO_BACKLOG, 1024)
.childOption(ChannelOption.TCP_NODELAY, true)
.childOption(ChannelOption.SO_KEEPALIVE, true)
// 此处有个大坑, 详见文章脱坑指南
.bind(port)
.sync();
if (channelFuture.isSuccess()) {
log.info("{} 启动成功", serverId);
redisTemplate.delete(RedisConfig.NETTY_SERVER_LOCK);
}
thisNodeHandle(port);
channelFuture.channel().closeFuture().sync();
} catch (InterruptedException exception) {
log.error("{} 启动失败: {}", serverId, exception.getMessage());
} finally {
redisTemplate.delete(RedisConfig.NETTY_SERVER_LOCK);
}
}
private void thisNodeHandle(int port) {
Set<String> nodeList = new HashSet<>();
Object nettyServerList = redisOperation.get(RedisConfig.NETTY_SERVER_SET);
if (nettyServerList != null) {
nodeList = new HashSet<>(JSON.parseArray(nettyServerList.toString(), String.class));
for (String hostAndPort : nodeList) {
String[] split = hostAndPort.split("@");
String connectHost = split[0];
if (serverId.equals(connectHost)) {
// 自己不应该连自己
continue;
}
String connectPort = split[1];
NettyClient nettyClient = new NettyClient();
nettyClient.setClientWorkerGroup(clientWorkerGroup);
nettyClient.setClientBusinessGroup(clientBusinessGroup);
NettyClientHandler nettyClientHandler = new NettyClientHandler();
nettyClientHandler.setUserCode(serverId);
nettyClientHandler.setHostName(connectHost);
nettyClientHandler.setPort(connectPort);
nettyClient.createClient(nettyClientHandler);
NettyStatic.NETTY_CLIENT_HANDLER.put(RedisConfig.key(connectHost, connectPort), nettyClientHandler);
NettyStatic.NETTY_CLIENT.put(nettyClientHandler, nettyClient);
}
}
nodeList.add(RedisConfig.key(serverId, port + ""));
redisOperation.set(RedisConfig.NETTY_SERVER_SET, JSON.toJSONString(nodeList));
}
public void stop() {
bossGroup.shutdownGracefully().syncUninterruptibly();
workerGroup.shutdownGracefully().syncUninterruptibly();
log.info("TCP服务关闭成功");
Object nettyServerList = redisOperation.get(RedisConfig.NETTY_SERVER_SET);
List<String> hostList = JSON.parseArray(nettyServerList.toString(), String.class);
hostList.remove(RedisConfig.key(serverId, port + ""));
if (CollectionUtils.isEmpty(hostList)) {
redisTemplate.delete(RedisConfig.NETTY_SERVER_SET);
} else {
redisOperation.set(RedisConfig.NETTY_SERVER_SET, JSON.toJSONString(hostList));
}
redisTemplate.delete(RedisConfig.key(serverId, port + ""));
}
@PreDestroy
public void destroy() {
stop();
}
}
2.5 修改 NettyServerHandler类
package com.hahashou.netty.server.config;
import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Component
@ChannelHandler.Sharable
@Slf4j
public class NettyServerHandler extends ChannelInboundHandlerAdapter {
@Value("${netty.server.id}")
private String serverId;
@Value("${netty.server.port}")
private int port;
public static String SERVER_PREFIX = "netty-server-";
@Resource
private RedisTemplate<String, Object> redisTemplate;
@Resource
private ValueOperations<String, Object> redisOperation;
@Override
public void channelActive(ChannelHandlerContext ctx) {
Channel channel = ctx.channel();
String channelId = channel.id().asLongText();
log.info("客户端连接, channelId : {}", channelId);
NettyStatic.CHANNEL.put(channelId, channel);
Message message = new Message();
message.setChannelId(channelId);
channel.writeAndFlush(Message.transfer(message));
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
String channelId = ctx.channel().id().asLongText();
log.info("客户端断开连接, channelId : {}", channelId);
NettyStatic.CHANNEL.remove(channelId);
for (Map.Entry<String, String> entry : NettyStatic.USER_CHANNEL.entrySet()) {
if (entry.getValue().equals(channelId)) {
redisTemplate.delete(entry.getKey());
break;
}
}
redisOperation.set(RedisConfig.key(serverId, port + ""), CONNECT_NUMBER.decrementAndGet());
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
if (msg != null) {
Message message = JSON.parseObject(msg.toString(), Message.class);
String userCode = message.getUserCode(),
channelId = message.getChannelId(),
friendUserCode = message.getFriendUserCode();
if (StringUtils.hasText(userCode) && StringUtils.hasText(channelId)) {
connect(userCode, channelId);
} else if (StringUtils.hasText(message.getText())) {
Object code = redisOperation.get(friendUserCode);
if (code != null) {
String queryServerId = code.toString();
message.setServerId(serverId.equals(queryServerId) ? null : queryServerId);
if (StringUtils.hasText(friendUserCode)) {
sendOtherClient(message);
} else {
sendAdmin(ctx.channel(), message);
}
} else {
offlineMessage(friendUserCode, message);
}
}
}
}
/** 后续还可以做一个最大连接数的限制, 到达后通知monitor, 使后续客户端不再连接此服务端 */
public static AtomicLong CONNECT_NUMBER = new AtomicLong(0);
/**
* 建立连接
* @param userCode
* @param channelId
*/
private void connect(String userCode, String channelId) {
log.info("{} 连接", userCode);
NettyStatic.USER_CHANNEL.put(userCode, channelId);
if (!userCode.startsWith(SERVER_PREFIX)) {
redisOperation.set(userCode, serverId);
}
redisOperation.set(RedisConfig.key(serverId, port + ""), CONNECT_NUMBER.incrementAndGet());
}
/**
* 发送给其他客户端
* @param message
*/
private void sendOtherClient(Message message) {
String friendUserCode = message.getFriendUserCode(),
serverId = message.getServerId();
String queryChannelId;
if (StringUtils.hasText(serverId)) {
log.debug("向" + serverId + " 进行转发");
queryChannelId = NettyStatic.USER_CHANNEL.get(serverId);
} else {
queryChannelId = NettyStatic.USER_CHANNEL.get(friendUserCode);
}
if (StringUtils.hasText(queryChannelId)) {
Channel channel = NettyStatic.CHANNEL.get(queryChannelId);
if (channel == null) {
offlineMessage(friendUserCode, message);
return;
}
channel.writeAndFlush(Message.transfer(message));
} else {
offlineMessage(friendUserCode, message);
}
}
/**
* 离线消息存储Redis
* @param friendUserCode
* @param message
*/
public void offlineMessage(String friendUserCode, Message message) {
// 1条message在redis中大概是100B, 1万条算1M, redis.conf的maxmemory设置的是256M
List<Message> messageList = new ArrayList<>();
Object offlineMessage = redisOperation.get(RedisConfig.OFFLINE_MESSAGE + friendUserCode);
if (offlineMessage != null) {
messageList = JSON.parseArray(offlineMessage.toString(), Message.class);
}
messageList.add(message);
redisOperation.set(RedisConfig.OFFLINE_MESSAGE + friendUserCode, JSON.toJSONString(messageList));
}
/**
* 发送给服务端
* @param channel
* @param message
*/
private void sendAdmin(Channel channel, Message message) {
message.setUserCode("ADMIN");
message.setText(LocalDateTime.now().toString());
channel.writeAndFlush(Message.transfer(message));
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
log.info("客户端发生异常");
}
}
2.6 修改 ServerService接口
package com.hahashou.netty.server.service;
import com.hahashou.netty.server.config.Message;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
public interface ServerService {
/**
* 发送消息
* @param dto
*/
void send(Message dto);
}
2.7 修改 ServerServiceImpl类
package com.hahashou.netty.server.service.impl;
import com.alibaba.fastjson.JSON;
import com.hahashou.netty.server.config.*;
import com.hahashou.netty.server.service.ServerService;
import io.netty.channel.Channel;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.data.redis.core.ValueOperations;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.security.crypto.password.PasswordEncoder;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import javax.annotation.Resource;
import java.util.*;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Service
@Slf4j
public class ServerServiceImpl implements ServerService {
@Value("${netty.server.id}")
private String serverId;
@Resource
private PasswordEncoder passwordEncoder;
@Resource
private ValueOperations<String, Object> redisOperation;
@Override
public void send(Message dto) {
String friendUserCode = dto.getFriendUserCode();
if (StringUtils.hasText(friendUserCode)) {
Object code = redisOperation.get(friendUserCode);
if (code != null) {
String queryServerId = code.toString();
dto.setServerId(serverId.equals(queryServerId) ? null : queryServerId);
if (StringUtils.hasText(friendUserCode)) {
sendOtherClient(dto);
}
} else {
offlineMessage(friendUserCode, dto);
}
} else {
// 全体广播, 需要校验秘钥(inputSecretKey应该是一个动态值, 通过手机+验证码每次广播时获取, 自行实现)
String inputSecretKey = dto.getSecretKey();
// encodedPassword生成见main方法
String encodedPassword = "$2a$10$J/UEqtme/w2D0TWB4gJKFeSsyc3s8pepr6ahzOsORkC9zpaLSvZbG";
if (StringUtils.hasText(inputSecretKey) && passwordEncoder.matches(inputSecretKey, encodedPassword)) {
dto.setSecretKey(null);
for (Map.Entry<String, String> entry : NettyStatic.USER_CHANNEL.entrySet()) {
String key = entry.getKey();
if (key.startsWith(NettyServerHandler.SERVER_PREFIX)) {
// 这里可以用http调用其他服务端, 自行补充(信息redis都有)
continue;
}
// 只处理连接本端的客户端
String value = entry.getValue();
Channel channel = NettyStatic.CHANNEL.get(value);
if (channel == null) {
offlineMessage(friendUserCode, dto);
return;
}
channel.writeAndFlush(Message.transfer(dto));
}
}
}
}
public static void main(String[] args) {
String text = "uTωAoJIGBcy7piYCFgQntVvEh8RH6WMU";
PasswordEncoder passwordEncoder = new BCryptPasswordEncoder();
String encode = passwordEncoder.encode(text);
log.info(encode);
if (passwordEncoder.matches(text, encode)) {
log.info("秘钥正确");
}
}
/**
* 发送给其他客户端
* @param message
*/
private void sendOtherClient(Message message) {
String friendUserCode = message.getFriendUserCode(),
serverId = message.getServerId();
String queryChannelId;
if (StringUtils.hasText(serverId)) {
log.info("向" + serverId + " 进行转发");
queryChannelId = NettyStatic.USER_CHANNEL.get(serverId);
} else {
queryChannelId = NettyStatic.USER_CHANNEL.get(friendUserCode);
}
if (StringUtils.hasText(queryChannelId)) {
Channel channel = NettyStatic.CHANNEL.get(queryChannelId);
if (channel == null) {
offlineMessage(friendUserCode, message);
return;
}
channel.writeAndFlush(Message.transfer(message));
} else {
offlineMessage(friendUserCode, message);
}
}
/**
* 离线消息存储Redis
* @param friendUserCode
* @param message
*/
public void offlineMessage(String friendUserCode, Message message) {
List<Message> messageList = new ArrayList<>();
Object offlineMessage = redisOperation.get(RedisConfig.OFFLINE_MESSAGE + friendUserCode);
if (offlineMessage != null) {
messageList = JSON.parseArray(offlineMessage.toString(), Message.class);
}
messageList.add(message);
redisOperation.set(RedisConfig.OFFLINE_MESSAGE + friendUserCode, JSON.toJSONString(messageList));
}
}
2.8 修改 ServerController类
package com.hahashou.netty.server.controller;
import com.hahashou.netty.server.config.Message;
import com.hahashou.netty.server.service.ServerService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@RestController
@RequestMapping("/server")
@Slf4j
public class ServerController {
@Resource
private ServerService serverService;
/**
* 秘钥记录: uTωAoJIGBcy7piYCFgQntVvEh8RH6WMU
* @param dto
* @return
*/
@PostMapping("/send")
public String send(@RequestBody Message dto) {
serverService.send(dto);
return "success";
}
/**
* 健康检查
*/
@GetMapping("/health")
public void health() {}
}
2.9 打2个jar包, netty-server-1和netty-server-2, 分别放在161 162
3. 客户端改造
3.1 config包下新增 ClientStatic类, 用户Code定义在此处
package com.hahashou.netty.client.config;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.util.HashedWheelTimer;
import io.netty.util.concurrent.EventExecutorGroup;
/**
* @description: 静态常量
* @author: 哼唧兽
* @date: 9999/9/21
**/
public class ClientStatic {
public static String USER_CODE = "Aa";
public static NettyClient CLIENT;
public static NioEventLoopGroup WORKER;
public static EventExecutorGroup BUSINESS;
public static NettyClientHandler CLIENT_HANDLER;
public static String IP;
public static int PORT;
private static HashedWheelTimer HASHED_WHEEL_SINGLE;
public static HashedWheelTimer hashedWheelSingle() {
if (HASHED_WHEEL_SINGLE == null) {
HASHED_WHEEL_SINGLE = new HashedWheelTimer();
}
return HASHED_WHEEL_SINGLE;
}
}
3.2 修改 application.yml
server:
port: 32001
logging:
level:
com.hahashou.netty: info
spring:
servlet:
multipart:
max-file-size: 128MB
max-request-size: 256MB
minio:
endpoint: http://192.168.109.160:9000
accessKey: root
secretKey: root123456
3.3 修改 类
package com.hahashou.netty.client.config;
import io.netty.bootstrap.Bootstrap;
import io.netty.channel.Channel;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.Timeout;
import io.netty.util.TimerTask;
import io.netty.util.concurrent.DefaultEventExecutorGroup;
import io.netty.util.concurrent.RejectedExecutionHandlers;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* @description: Netty-TCP服务
* @author: 哼唧兽
* @date: 9999/9/21
**/
@Slf4j
public class NettyClient implements TimerTask {
public static Channel CHANNEL;
public NettyClient() {
try {
latestServer();
createClient();
} catch (IOException exception) {
log.error("连接monitor服务异常: {}, 10s后重试", exception.getMessage());
ClientStatic.hashedWheelSingle().newTimeout(this, 10, TimeUnit.SECONDS);
} catch (InterruptedException exception) {
log.error("连接netty异常: {}, 5s后重试", exception.getMessage());
ClientStatic.hashedWheelSingle().newTimeout(this, 5, TimeUnit.SECONDS);
}
}
private void latestServer() throws IOException {
// 也可引入第三方库请求
// 实际场景中应该使用域名去请求, 通过Nginx维护两三个服务就可以了
URL url = new URL("http://192.168.109.160:31000/nettyServer");
HttpURLConnection httpConnection = (HttpURLConnection) url.openConnection();
httpConnection.setRequestMethod("GET");
int responseCode = httpConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(httpConnection.getInputStream()));
String inputLine;
StringBuffer responseBuffer = new StringBuffer();
while ((inputLine = reader.readLine()) != null) {
responseBuffer.append(inputLine);
}
reader.close();
String response = responseBuffer.toString();
if (StringUtils.hasText(response)) {
String[] split = response.split("@");
ClientStatic.IP = split[0];
ClientStatic.PORT = Integer.parseInt(split[1]);
} else {
throw new IOException("无可用的netty服务端");
}
} else {
throw new IOException("请求服务端信息失败, responseCode: " + responseCode);
}
}
private void createClient() throws InterruptedException {
ClientStatic.WORKER = new NioEventLoopGroup(4);
ClientStatic.BUSINESS = new DefaultEventExecutorGroup(1,
new BusinessThreadFactory(), 100000, RejectedExecutionHandlers.reject());
ClientStatic.CLIENT_HANDLER = new NettyClientHandler();
Bootstrap bootstrap = new Bootstrap();
bootstrap.group(ClientStatic.WORKER)
.channel(NioSocketChannel.class)
.handler(new ChannelInitializer<NioSocketChannel>() {
@Override
protected void initChannel(NioSocketChannel ch) throws Exception {
ChannelPipeline pipeline = ch.pipeline();
pipeline.addLast(new StringDecoder(Charset.forName("UTF-8")));
pipeline.addLast(new StringEncoder(Charset.forName("UTF-8")));
pipeline.addLast(ClientStatic.BUSINESS, ClientStatic.CLIENT_HANDLER);
}});
CHANNEL = bootstrap.connect(ClientStatic.IP, ClientStatic.PORT).sync().channel();
}
public void stopClient() {
ClientStatic.WORKER.shutdownGracefully().syncUninterruptibly();
ClientStatic.WORKER = null;
ClientStatic.BUSINESS = null;
ClientStatic.CLIENT_HANDLER = null;
log.info("客户端关闭成功");
}
@Override
public void run(Timeout timeout) {
ClientStatic.CLIENT = new NettyClient();
}
static class BusinessThreadFactory implements ThreadFactory {
private final ThreadGroup group;
private final AtomicInteger threadNumber = new AtomicInteger(1);
private final String namePrefix;
BusinessThreadFactory() {
SecurityManager securityManager = System.getSecurityManager();
group = securityManager != null ? securityManager.getThreadGroup() : Thread.currentThread().getThreadGroup();
namePrefix = "client-thread-";
}
@Override
public Thread newThread(Runnable r) {
Thread thread = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0);
if (thread.isDaemon()) {
thread.setDaemon(false);
}
if (thread.getPriority() != Thread.NORM_PRIORITY) {
thread.setPriority(Thread.NORM_PRIORITY);
}
return thread;
}
}
}
3.4 修改 NettyClientHandler类
package com.hahashou.netty.client.config;
import com.alibaba.fastjson.JSON;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@ChannelHandler.Sharable
@Slf4j
public class NettyClientHandler extends ChannelInboundHandlerAdapter {
public static Channel CHANNEL;
@Override
public void channelActive(ChannelHandlerContext ctx) {
CHANNEL = ctx.channel();
log.info("客户端 " + ClientStatic.USER_CODE + " 上线");
}
@Override
public void channelInactive(ChannelHandlerContext ctx) {
CHANNEL = null;
log.error("服务端已下线, 准备重连");
ClientStatic.CLIENT = new NettyClient();
}
@Override
public void channelRead(ChannelHandlerContext ctx, Object msg) {
if (msg != null) {
Message message = JSON.parseObject(msg.toString(), Message.class);
String channelId = message.getChannelId(),
text = message.getText();
if (StringUtils.hasText(channelId)) {
Channel channel = ctx.channel();
message.setUserCode(ClientStatic.USER_CODE);
channel.writeAndFlush(Message.transfer(message));
} else if (StringUtils.hasText(text)) {
log.info("收到" + message.getUserCode() + "消息: {}", text);
}
}
}
@Override
public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
CHANNEL = null;
}
}
3.5 之前ClientServiceImpl包名错了, 改成 impl
3.6 修改 ClientController类
package com.hahashou.netty.client.controller;
import com.alibaba.fastjson.JSON;
import com.hahashou.netty.client.config.*;
import com.hahashou.netty.client.service.ClientService;
import io.netty.channel.Channel;
import lombok.extern.slf4j.Slf4j;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.*;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
/**
* @description:
* @author: 哼唧兽
* @date: 9999/9/21
**/
@RestController
@RequestMapping("/client")
@Slf4j
public class ClientController {
@Resource
private ClientService clientService;
@PostMapping("/send")
public String send(@RequestBody Message dto) {
Channel channel = NettyClientHandler.CHANNEL;
if (channel == null) {
return "服务端已下线";
}
channel.writeAndFlush(Message.transfer(dto));
return "success";
}
@GetMapping("/upload/{userCode}")
public String upload(@PathVariable String userCode, final HttpServletRequest httpServletRequest) {
if (StringUtils.isEmpty(userCode)) {
return "userCode is null";
}
Message upload = clientService.upload(userCode, httpServletRequest);
return JSON.toJSONString(upload);
}
@GetMapping("/link")
public String link(@RequestParam String fileName) {
// 如果Bucket包含多级目录, fileName为Bucket下文件的全路径名
if (StringUtils.isEmpty(fileName)) {
return "fileName is null";
}
return clientService.link(fileName);
}
@GetMapping("/start")
public void start() {
ClientStatic.CLIENT = new NettyClient();
}
@GetMapping("/stop")
public String stopClient() {
ClientStatic.CLIENT.stopClient();
ClientStatic.CLIENT = null;
return "success";
}
}
3.7 删除 EventLoopGroupConfig类
4. 测试
4.1 本地启动客户端Aa, 调用/client/start接口, 连接服务端
此时, 因为没有服务端, 所有会一直重试
4.2 此时启动netty-server-1
客户端稍后会连接到服务端
4.3 此时启动netty-server-2
此时如果请求: 160在线服务端 , 会返回 192.168.109.162@35000, 因为负载情况如下
4.4 直接kill掉netty-server-1(非正常断线)
可以看到, Aa已经能正常连接到服务端netty-server-2, 同时redis里修改成只有netty-server-2在线上