mq消费handle中的方法部分代码
// 队列处理
if (ElasticSearchEnums.ES_UPDATE.getMqType().equalsIgnoreCase(mqType)) {
return loanEsQueueService.updateLoanToEs(object);
} else if (ElasticSearchEnums.ES_INSERT.getMqType().equalsIgnoreCase(mqType)) {
return loanEsQueueService.saveLoanToEs(object);
}
消息累积以及定时
思路:
compareSizeAndSleep 方法:
1.根据上一层方法先添加到对应的类型队列中去
2.进行队列的数量判断和当前休眠次数判断(相当于休眠时间,定时)
3. 新起线程池去执行 saveLoanListToEs 方法。
4. 获取执行后的结果,从map中。
5. 判断是否执行成功。
这里的话,前面进来的mq线程会在这里hold住,当该方法中代码都执行完后,对应的mq线程会根据mq消息id之类的标识,从map中取到自己那条消息的执行结果,没有则说明执行失败,返回false。
saveLoanListToEs 方法:
1.将队列中的任务进行一个个弹出,并同时将结果添加到map中。
2.进行es执行
3.判断es结果,如果失败,则将前面添加的标识从map中移除。
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import javax.annotation.Resource;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
/**
* @author dandanforgetlove
* @description
* @date 2022/2/28
*/
@Log4j2
@Component
public class LoanEsQueueServiceImpl implements LoanEsQueueService {
@Value("${loan.es.bulk.size:500}")
long loanEsBulkSize;
@Value("${loan.es.bulk.sleep.number:100}")
long loanEsBulkSleepNumber;
@Value("${loan.es.bulk.fixed.thread.pool:1}")
static int loanEsBulkFixedThreadPoolNum;
public static volatile LinkedBlockingQueue<LoanEntity> loanSaveLinkedBlockingQueue = new LinkedBlockingQueue<>(1000);
public static volatile LinkedBlockingQueue<LoanEntity> loanUpdateLinkedBlockingQueue = new LinkedBlockingQueue<>(1000);
public static volatile Map<String, Boolean> loanEsResultMap = new Hashtable<>();
@Resource
private LoanElasticSearchService loanElasticSearchService;
@Override
public boolean saveLoanToEs(LoanEntity loanEntity){
return compareSizeAndSleep(ElasticSearchEnums.ES_LOAN_INSERT.getMqType(), loanSaveLinkedBlockingQueue, loanEntity);
}
@Override
public boolean updateLoanToEs(LoanEntity loanEntity){
return compareSizeAndSleep(ElasticSearchEnums.ES_LOAN_UPDATE.getMqType(), loanUpdateLinkedBlockingQueue, loanEntity);
}
public boolean compareSizeAndSleep(String opera, LinkedBlockingQueue<LoanEntity> linkedBlockingQueue, LoanEntity loanEntity){
try {
// 自旋指定次数
int sleepNum = 0;
// 先添加到对应的队列中,判断队列是否达到指定大小,没有则进行自旋
linkedBlockingQueue.add(loanEntity);
while(linkedBlockingQueue.size() < loanEsBulkSize && sleepNum < loanEsBulkSleepNumber){
sleepNum++;
Thread.sleep(10);
}
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(loanEsBulkFixedThreadPoolNum);
// 新起线程执行
fixedThreadPool.execute(()->{
saveLoanListToEs(opera, linkedBlockingQueue, loanEntity);
});
Boolean remove = loanEsResultMap.remove(loanEntity.getLoanNo() + opera);
fixedThreadPool.shutdown();
// 从map中获取是否成功
return remove == null ? false : remove;
} catch (InterruptedException e) {
log.error("LoanEsQueueServiceImpl#compareSizeAndSleep 执行异常", e);
}
return false;
}
/**
* 进行批处理
* 如果失败,则清空map
* @author dongdong
* @date 2022/3/1
* @param
* @return
*/
synchronized boolean saveLoanListToEs(String opera, LinkedBlockingQueue<LoanEntity> linkedBlockingQueue, LoanEntity loanEntity){
try {
List<LoanEntity> loanEntityList = new ArrayList<>();
for(int i = 0; i < loanEsBulkSize; i++){
LoanEntity poll = linkedBlockingQueue.poll();
if (poll != null) {
loanEntityList.add(poll);
loanEsResultMap.put(poll.getLoanNo() + opera, true);
}
}
if(loanEntityList.size() == 0){
return false;
}
long result = 0;
if (ElasticSearchEnums.ES_LOAN_UPDATE.getMqType().equalsIgnoreCase(opera)) {
result = loanElasticSearchService.updateLoanToElasticSearch(loanEntityList);
} else if (ElasticSearchEnums.ES_LOAN_INSERT.getMqType().equalsIgnoreCase(opera)) {
result = loanElasticSearchService.insertLoanToElasticSearch(loanEntityList);
}
if(result == 0){
loanEntityList.stream().forEach(loan -> loanEsResultMap.remove(loan.getLoanNo() + opera));
}
return true;
} catch (Exception e) {
log.error("LoanEsQueueServiceImpl#saveLoanListToEs 执行异常", e);
}
return false;
}
}