使用多线程推松数据到ES中,无数据丢失

最新推荐文章于 2024-07-25 16:37:16 发布

CatServer

最新推荐文章于 2024-07-25 16:37:16 发布

阅读量188

点赞数

文章标签： elasticsearch java spring

本文链接：https://blog.csdn.net/CuiGongZi/article/details/133180365

版权

ES作为缓存,越来越流行,使用多线程推送数据方案

public void bulkImportAll(String year,Integer source){
		//初始化线程池,
        ThreadPoolExecutor executor = new ThreadPoolExecutor(16, 32, 20, TimeUnit.MILLISECONDS,
                new ArrayBlockingQueue<>(1024), new ThreadPoolExecutor.CallerRunsPolicy());
        int count = orderDao.selectOrderCount(StringUtils.isEmpty(year)?DateUtil.getYear():year,source);
		//每个线程发配的任务数
        int size = 1000;
        //需要查询次数
        int times = count / size;
        if (count % size != 0) {
            times = times + 1;
        }
        for (int i = 0; i < times; i++) {
            BatchSelectOrder batchSelectOrder = new BatchSelectOrder(source,year,i,size,restHighLevelClient,orderDao,orderItemDao);
            executor.execute(batchSelectOrder);
            log.info("线程池中线程数目:{} ,队列中等待执行的任务数目:{} ,已执行完的任务数目：{}" , executor.getPoolSize(),executor.getQueue().size(),
                    executor.getCompletedTaskCount());
        }
        executor.shutdown();
    }



@Slf4j
public class BatchSelectOrder implements Runnable {


    private static String indexName="order_es_analyse";

    private int bindex;//当前页数

    private int size;//每页查询多少条

    private String year ;//年份

    private Integer source ;//卖场

    private RestHighLevelClient restHighLevelClient;

    private OrderDao orderDao;

    private OrderItemDao orderItemDao;


    public BatchSelectOrder(int bindex, int size) {
        this.bindex = bindex;
        this.size = size;
    }

    public BatchSelectOrder(Integer source,String year,int bindex, int size, RestHighLevelClient restHighLevelClient, OrderDao orderDao, OrderItemDao orderItemDao) {
        this.source = source;
        this.year = year;
        this.bindex = bindex;
        this.size = size;
        this.restHighLevelClient = restHighLevelClient;
        this.orderDao = orderDao;
        this.orderItemDao = orderItemDao;
    }

    
    public void batchSelectOrders() {
        log.info("begin start batch select task....");
        //查询主表数据
        List<OrderAnalyzes> ordersByYear = orderDao.findOrdersByYear(StringUtils.isEmpty(year)?DateUtil.getYear():year,source,bindex* size, size);
        if (CollectionUtils.isNotEmpty(ordersByYear)){
            ordersByYear.forEach(c-> {
                  //查询子表数据
                List<OrderItemAnalyzes> orderItems = orderItemDao.findOrderItemByOrderId(StringUtils.isEmpty(year)?DateUtil.getYear():year, c.getTradeId());
                c.setOrderItemEntityList(orderItems);
            });
            List<OrderESEntity> orderEsEntities =new ArrayList<>();
                //处理数据,把查询到数据赋值到ES的实体类中
            ordersByYear.forEach(c->{
                OrderESEntity orderEsEntity =new OrderESEntity();
                exchangeEs(StringUtils.isEmpty(year)?DateUtil.getYear():year,c,orderEsEntity);
                orderEsEntities.add(orderEsEntity);
            });
            //创建两个请求,第一个请求用来添加数据,第二个请求用来删除数据
            BulkRequest bulkRequest = new BulkRequest();
            BulkRequest bulkRequestForDel = new BulkRequest();

            orderEsEntities.forEach(v->{
                if(v.getDeleted()){
                    DeleteRequest deleteRequest = new DeleteRequest(indexName,v.getId());
                    bulkRequestForDel.add(deleteRequest);
                }else{
                    IndexRequest indexRequest=new IndexRequest(indexName);
                    indexRequest.id(v.getId()).source(JSON.toJSONString(v), XContentType.JSON);
                    bulkRequest.add(indexRequest);
                }
            });
            restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT,new ActionListener<BulkResponse>() {
                @Override
                public void onResponse(BulkResponse bulkResponse) {
                    log.info("批量写入es成功");
                }
                @Override
                public void onFailure(Exception e) {
                    //出错的时候调用
                    log.info("批量写入es错误："+e.getMessage());
                }
            });
            if(bulkRequestForDel.requests().size()>0){
                restHighLevelClient.bulkAsync(bulkRequestForDel, RequestOptions.DEFAULT, new ActionListener<BulkResponse>() {
                    @Override
                    public void onResponse(BulkResponse bulkItemResponses) {
                        log.info("按照id批量删除部分数据成功");
                    }

                    @Override
                    public void onFailure(Exception e) {
                        log.info("按照id批量删除部分数据失败："+e.getMessage());
                    }
                });
            }

            log.info("end batch select task....");
        }
    }



    @Override
    public void run() {
        batchSelectOrders() ;
    }
}