ES作为缓存,越来越流行,使用多线程推送数据方案
public void bulkImportAll(String year,Integer source){
ThreadPoolExecutor executor = new ThreadPoolExecutor(16, 32, 20, TimeUnit.MILLISECONDS,
new ArrayBlockingQueue<>(1024), new ThreadPoolExecutor.CallerRunsPolicy());
int count = orderDao.selectOrderCount(StringUtils.isEmpty(year)?DateUtil.getYear():year,source);
int size = 1000;
int times = count / size;
if (count % size != 0) {
times = times + 1;
}
for (int i = 0; i < times; i++) {
BatchSelectOrder batchSelectOrder = new BatchSelectOrder(source,year,i,size,restHighLevelClient,orderDao,orderItemDao);
executor.execute(batchSelectOrder);
log.info("线程池中线程数目:{} ,队列中等待执行的任务数目:{} ,已执行完的任务数目:{}" , executor.getPoolSize(),executor.getQueue().size(),
executor.getCompletedTaskCount());
}
executor.shutdown();
}
@Slf4j
public class BatchSelectOrder implements Runnable {
private static String indexName="order_es_analyse";
private int bindex;
private int size;
private String year ;
private Integer source ;
private RestHighLevelClient restHighLevelClient;
private OrderDao orderDao;
private OrderItemDao orderItemDao;
public BatchSelectOrder(int bindex, int size) {
this.bindex = bindex;
this.size = size;
}
public BatchSelectOrder(Integer source,String year,int bindex, int size, RestHighLevelClient restHighLevelClient, OrderDao orderDao, OrderItemDao orderItemDao) {
this.source = source;
this.year = year;
this.bindex = bindex;
this.size = size;
this.restHighLevelClient = restHighLevelClient;
this.orderDao = orderDao;
this.orderItemDao = orderItemDao;
}
public void batchSelectOrders() {
log.info("begin start batch select task....");
List<OrderAnalyzes> ordersByYear = orderDao.findOrdersByYear(StringUtils.isEmpty(year)?DateUtil.getYear():year,source,bindex* size, size);
if (CollectionUtils.isNotEmpty(ordersByYear)){
ordersByYear.forEach(c-> {
List<OrderItemAnalyzes> orderItems = orderItemDao.findOrderItemByOrderId(StringUtils.isEmpty(year)?DateUtil.getYear():year, c.getTradeId());
c.setOrderItemEntityList(orderItems);
});
List<OrderESEntity> orderEsEntities =new ArrayList<>();
ordersByYear.forEach(c->{
OrderESEntity orderEsEntity =new OrderESEntity();
exchangeEs(StringUtils.isEmpty(year)?DateUtil.getYear():year,c,orderEsEntity);
orderEsEntities.add(orderEsEntity);
});
BulkRequest bulkRequest = new BulkRequest();
BulkRequest bulkRequestForDel = new BulkRequest();
orderEsEntities.forEach(v->{
if(v.getDeleted()){
DeleteRequest deleteRequest = new DeleteRequest(indexName,v.getId());
bulkRequestForDel.add(deleteRequest);
}else{
IndexRequest indexRequest=new IndexRequest(indexName);
indexRequest.id(v.getId()).source(JSON.toJSONString(v), XContentType.JSON);
bulkRequest.add(indexRequest);
}
});
restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT,new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkResponse) {
log.info("批量写入es成功");
}
@Override
public void onFailure(Exception e) {
log.info("批量写入es错误:"+e.getMessage());
}
});
if(bulkRequestForDel.requests().size()>0){
restHighLevelClient.bulkAsync(bulkRequestForDel, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() {
@Override
public void onResponse(BulkResponse bulkItemResponses) {
log.info("按照id批量删除部分数据成功");
}
@Override
public void onFailure(Exception e) {
log.info("按照id批量删除部分数据失败:"+e.getMessage());
}
});
}
log.info("end batch select task....");
}
}
@Override
public void run() {
batchSelectOrders() ;
}
}