在计算机中,当消费者消费的速度无法满足接收消息的速度时,通常会通过缓存来解决这个问题,L1,L2,L3等,其根本原因在于磁盘写入的速度远远慢于内存操作的速度,虽然固态可以在一定程度上缓解这个问题,但是问题依然是存在的。在Java服务端也会存在这样的问题,通常解决思路也是一样的,废话不说 ,提码来见;
package com.falcon.design.service;
import com.falcon.design.attendance.domain.Message;
import com.falcon.design.utils.SpringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
public class MessageHandler {
private static volatile MessageHandler handler = null;
private static int queueSize = 10000;
private IMessageService messageService = null;
private BlockingDeque<Message> insertQueue = new LinkedBlockingDeque<>(queueSize);
private BlockingDeque<Message> updateQueue = new LinkedBlockingDeque<>(queueSize);
private ExecutorService executorService = Executors.newFixedThreadPool(2);
public MessageHandler() {
messageService = (IMessageService) SpringUtils.getBean(IMessageService.class);
} public static MessageHandler getInstance(){
if(null == handler){
synchronized (MessageHandler.class){
if(null == handler){
handler = new MessageHandler();
handler.start(); } } } return handler;
} public void start(){
executorService.submit( (Runnable)() ->{ List<Message> messageList = null;
while (true){
messageList = new ArrayList<>();
try{
//阻塞直到拿到消息
Message takeMessage = insertQueue.take();
messageList.add(takeMessage);
long firsTime = System.currentTimeMillis();
for (int i = 0; i < 500; i++) {
long currTime = System.currentTimeMillis();
if(currTime - firsTime >5000){
break;
}
Message pollMessage = insertQueue.poll();
if(null == pollMessage){
continue;
}
messageList.add(pollMessage);
}
}catch (Exception e){
System.out.println("insert error "+e.getMessage());
}
//防止cpu占用率搞
try {
Thread.sleep(1);
}catch (Exception e){
System.out.println(e.getMessage());
}
if(messageList.isEmpty()){
continue;
}
try {
messageService.batchInsert(messageList);
System.out.println("insert database success !");
} catch (Exception e) {
System.out.println("insert database failed:"+e.getMessage());
}
}
});
executorService.submit( (Runnable)() ->{
List<Message> messageList = null;
while (true){
messageList = new ArrayList<>();
try{
//阻塞直到拿到消息
Message takeMessage = updateQueue.take();
messageList.add(takeMessage);
long firsTime = System.currentTimeMillis();
for (int i = 0; i < 500; i++) {
long currTime = System.currentTimeMillis();
if(currTime - firsTime >5000){
break;
}
Message pollMessage = insertQueue.poll();
if(null == pollMessage){
continue;
}
messageList.add(pollMessage);
}
}catch (Exception e){
System.out.println("insert error "+e.getMessage());
}
//防止cpu占用率搞
try {
Thread.sleep(1);
}catch (Exception e){
System.out.println(e.getMessage());
}
if(messageList.isEmpty()){
continue;
}
try {
messageService.batchUpdate(messageList);
System.out.println("update database success !");
} catch (Exception e) {
System.out.println("update database failed:"+e.getMessage());
}
}
});
}
public void putMsgInsert(Message message) {
monitorQueue(insertQueue, "insertQueue");
insertQueue.add(message);
}
public void putMsgUpdate(Message message){
monitorQueue(updateQueue, "updateQueue");
updateQueue.add(message);
}
private void monitorQueue(BlockingQueue<?> queue, String keyWord){
if(queue.size() >= queueSize){
queue.poll();
System.out.println(String.format("%s queue is full ,throw data", keyWord));
}
}
public static void main(String[] args) {
MessageHandler.getInstance().putMsgInsert(new Message());
MessageHandler.getInstance().putMsgUpdate(new Message());
}
}
在这里我用到了线程池和阻塞队列技术,测试发现这个核心逻辑可以保证每秒1000条数据的消费量,如果各位有什么不同的看法欢迎在评论区讨论。
相关的代码已在gitee进行了分享:git@gitee.com:fortunamajor/design.git