之前接口一直是一套代码从头执行到尾,在小型应用中可能无伤大雅,而在大批量要访问数据库等操作中,可能效率太低。本文给出了一个较“轻”的一个多线程实现的解决方案:
接口的背景是spring结合activemq,消费mq。现在要实现从mq里获取的消息,持久化进数据库。数据量比较大,可能是几万条数据量。
整体的开发思想是:
项目启动时,spring控制启动一个核心线程去拿内存队列(mqQueue)中的一定量的内容,并且在run中持久化,然后每次观察获取到的量,如果到达一定值,就再起一个线程去做同样的事。如果发现内容为空,则核心线程sleep,后面起的非核心线程之间break出run方法(作为注销)。
每次从MQ中获取到消息,就往mqQueue中offer,或者put进去。
public class ActiveBasePushListener implements MessageListener{
private static final Log Logger = LogFactory.getLog(ActiveBasePushListener.class);
@Autowired
private HistoryUtil historyUtil;
@Override
public void onMessage(Message message) {
try {
if (message instanceof TextMessage){
//当传Text类型的消息时启用,现不启用
}else if (message instanceof MapMessage){
MapMessage mapMessage = (MapMessage)message;
MQContent mqContent = msgToMQContent(mapMessage);
historyUtil.handle(mqContent);
}
} catch (Exception e) {
Logger.error("", e);
} finally {
try {
message.acknowledge();
} catch (JMSException e) {
Logger.info("ActiveMQBasePushListener_acknowledge_error",e);
}
}
}
其中的msgToMQContent方法只是将消息内容做一下转化,不影响对代码的阅读。这边主要是historyUtil.handle()方法,作为入口。
@Service
public class HistoryUtil {
@Autowired
private SqlSessionTemplate sqlSessionTemplate;
private static final Log logger = LogFactory.getLog(HistoryUtil.class);
private static ExecutorService executorService = Executors.newCachedThreadPool();
private static AtomicInteger threadNum = new AtomicInteger(0);
private static LinkedBlockingQueue<MQContent> mqQueue = new LinkedBlockingQueue<MQContent>(50000);
private static Integer maxConsumerCount = 30;
private static int batchMaxCount = 500;
@PostConstruct
private void initcfg() {
executorService.submit(new PushThread(true));
logger.info("初始化核心线程");
}
public void handle(MQContent vo) {
if (vo == null) return;
try {
mqQueue.put(vo);
} catch (Exception e) {
logger.error("QueuePushService_handle_error!", e);
}
}
public void addConsume() {
if (threadNum.get() < maxConsumerCount) {
PushThread pushThread = new PushThread(false);
executorService.submit(pushThread);
threadNum.incrementAndGet();
}
}
class PushThread implements Runnable {
private boolean coreThread;
/**
* 用于while循环停止
*/
private boolean flag = true;
private List<MQContent> list = new ArrayList<MQContent>();
/**
* 是否核心线程,至少有一个线程在消费
*
* @param coreThread
*/
public PushThread(boolean coreThread) {
this.coreThread = coreThread;
}
@Override
public void run() {
while (flag) {
try {
//一秒钟没数据,则线程休眠1秒
mqQueue.drainTo(list, batchMaxCount);
if (CollectionUtils.isEmpty(list)) {
if (coreThread) {
Thread.sleep(100);
continue;
} else {
threadNum.decrementAndGet();
logger.info("副线程break,副线程数为"+threadNum.get());
break;
}
}
if(list!=null&&list.size()>0){
int count = batchInsertTtHis(list);
if(coreThread){
logger.info("在主线程中写入数据库一次"+count+"条已上传");
}else{
logger.info("在副线程中写入数据库一次"+count+"条已上传");
}
}
//数据大于50,增加线程数。
if (list.size() >50 ) {
addConsume();
logger.info("起一个副线程,副线程数为"+threadNum.get());
}
list.clear();
} catch (Exception e) {
logger.error("QueuePushService_PushThread_run_error " , e);
}
}
}
public int batchInsertTtHis(List<MQContent> list){
int count=0;
int batchCount=0;
SqlSession session = sqlSessionTemplate.getSqlSessionFactory().openSession(ExecutorType.BATCH, false);
try {
for (MQContent mqContent : list) {
batchCount++;
count = count + 1;
session.insert("MQContentMapper.insert", mqContent);
if (batchCount >= 1000) {//到达1000条提交一次
session.commit();
batchCount = 0;
}
}
session.commit();
session.clearCache();
}catch (Exception e) {
logger.error("微信推送写入历史错误");
// 没有提交的数据可以回滚
session.rollback();
} finally {
session.close();
}
return count;
}
}
}