并发插入数据百万条

最新推荐文章于 2024-07-12 06:58:18 发布

u013045102

最新推荐文章于 2024-07-12 06:58:18 发布

阅读量768

点赞数

本文链接：https://blog.csdn.net/u013045102/article/details/90476448

版权

1、新闻并发入库数据

@PostMapping(value = "/api/testnews")
@ApiOperation(value = "测试添加新闻动态")
public void testcreate(Integer start,Integer count) {
    this.newsService.testTo161(start,count);
}

/**
 * 测试大批量添加新闻动态接口
 * @param count
 * @param start
 */
void testTo161(Integer start,Integer count);

public void testTo161(Integer start,Integer count) {
    long stime = System.currentTimeMillis();
    ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 10, 60,
            TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(10));

    executor.execute(() -> execAdd(start,count));
    executor.execute(() -> execAdd(start+count+1,count));
    executor.execute(() -> execAdd(start+2*count+1,count));
    executor.execute(() -> execAdd(start+3*count+1,count));
    executor.execute(() -> execAdd(start+4*count+1,count));
    executor.execute(() -> execAdd(start+5*count+1,count));
    executor.execute(() -> execAdd(start+6*count+1,count));
    executor.execute(() -> execAdd(start+7*count+1,count));
    executor.execute(() -> execAdd(start+8*count+1,count));
    executor.execute(() -> execAdd(start+9*count+1,count));
    executor.shutdown();

    logger.info("实际执行任务的线程数"+executor.getActiveCount());
    long etime = System.currentTimeMillis();
    logger.info("耗时" + (etime - stime) / 1000 + "秒");
}

private void execAdd(Integer start,Integer count){
    for (int i = start; i < start+count; i++) {
        News news = new News();
        news.setId(UUID.randomUUID().toString());
        news.setTitle("新闻标题"+i);
        news.setAuthor("新闻作者刘"+i+"某");
        news.setPublishStatus(1);
        news.setSource("新闻来源"+i);
        news.setContent("新闻正文"+i);
        news.setTopOrder(0);
        news.setCreateTime(new Date());
        news.setModifyTime(new Date());
        String json = gson.toJson(news, News.class);
        client.prepareIndex()
                .setIndex("lawfullyadministration.dynamicnewstest")
                .setType(TypeConstant.TYPE)
                .setId(news.getId())
                .setSource(json)
                .execute()
                .actionGet();
        logger.info(news.getId() + "入库161完成");
        logger.info("当前入库第"+i+"条记录");
        if(i == 100000){
            break;
        }
    }
}

2、政策并发入库数据

@PostMapping(value = "/api/testpolicys")
@ApiOperation(value = "测试添加政策")
public void testcreate(Integer start,Integer count) {
    this.policyService.testTo161(start,count);
}

/**
 * 测试大批量添加接口
 * @param count
 * @param start
 */
void testTo161(Integer start,Integer count);

public void testTo161(Integer start,Integer count) {
    long stime = System.currentTimeMillis();
    ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 10, 60,
            TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(10));

    executor.execute(() -> execAdd(start,count));
    executor.execute(() -> execAdd(start+count+1,count));
    executor.execute(() -> execAdd(start+2*count+1,count));
    executor.execute(() -> execAdd(start+3*count+1,count));
    executor.execute(() -> execAdd(start+4*count+1,count));
    executor.execute(() -> execAdd(start+5*count+1,count));
    executor.execute(() -> execAdd(start+6*count+1,count));
    executor.execute(() -> execAdd(start+7*count+1,count));
    executor.execute(() -> execAdd(start+8*count+1,count));
    executor.execute(() -> execAdd(start+9*count+1,count));
    executor.shutdown();

    logger.info("实际执行任务的线程数"+executor.getActiveCount());
    long etime = System.currentTimeMillis();
    logger.info("耗时" + (etime - stime) / 1000 + "秒");
}

private void execAdd(Integer start,Integer count){
    for (int i = start; i < start+count; i++) {
        if(i == 1000000){
            break;
        }
        Policy policy = new Policy();
        policy.setTitle("政策标题" + i);
        policy.setId(UUID.randomUUID().toString());
        policy.setArticleNumber("北京人力资源保障局[2019]" + i + "号");
        policy.setDraftOffice("" + i);
        policy.setPublishStatus(1);
        policy.setPublishDate(new Date());
        policy.setContent("正文" + i);
        policy.setFileId(("文件名" + i).hashCode() + "");
        policy.setFileName("文件名" + i+".doc");
        policy.setPerformDate(new Date());
        policy.setTopicCategory("topic" + i);
        policy.setValidity("现行有效");
        policy.setValidityGrade("地方性法规");
        policy.setCreateTime(new Date());
        policy.setModifyTime(new Date());
        String json = gson.toJson(policy, Policy.class);
        client.prepareIndex()
                .setIndex(/*IndexConstant.POLICY*/"lawfullyadministration.policytest")
                .setType(TypeConstant.TYPE)
                .setId(policy.getId())
                .setSource(json)
                .execute()
                .actionGet();
        logger.info(policy.getId() + "入库161完成");
        logger.info("当前入库第"+policy.getDraftOffice()+"条记录");
    }
}

3、报告并发入库数据

@PostMapping(value = "/api/testreports")
@ApiOperation(value = "测试添加报告")
public void testcreate(Integer start,Integer count) {
    this.reportService.testTo78(start,count);
}

/**
 * 测试大批量添加接口
 * @param count
 * @param start
 */
void testTo78(Integer start,Integer count);

private static final Logger logger = LoggerFactory.getLogger(ReportServiceImpl.class);

@Override
public void testTo78(Integer start,Integer count) {
    long stime = System.currentTimeMillis();
    ThreadPoolExecutor executor = new ThreadPoolExecutor(10, 10, 60,
            TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(10));

    executor.execute(() -> execAdd(start,count));
    executor.execute(() -> execAdd(start+count+1,count));
    executor.execute(() -> execAdd(start+2*count+1,count));
    executor.execute(() -> execAdd(start+3*count+1,count));
    executor.execute(() -> execAdd(start+4*count+1,count));
    executor.execute(() -> execAdd(start+5*count+1,count));
    executor.execute(() -> execAdd(start+6*count+1,count));
    executor.execute(() -> execAdd(start+7*count+1,count));
    executor.execute(() -> execAdd(start+8*count+1,count));
    executor.execute(() -> execAdd(start+9*count+1,count));
    executor.shutdown();

    logger.info("实际执行任务的线程数"+executor.getActiveCount());
    long etime = System.currentTimeMillis();
    logger.info("耗时" + (etime - stime) / 1000 + "秒");
}

private void execAdd(Integer start,Integer count){
    for (int i = start; i < start+count; i++) {
        if(i == 1000000){
            break;
        }
        Report report = new Report();
        report.setTitle("报告标题" + i);
        report.setId(UUID.randomUUID().toString());
        report.setPublishStatus(1);
        report.setTopOrder(0);
        report.setSource("报告来源"+i);
        report.setAuthor("报告作者"+i);
        report.setContent("报告正文" + i);
        report.setFileId(("报告文件名" + i).hashCode() + "");
        report.setFileName("报告文件名" + i+".doc");
        report.setCreateTime(new Date());
        report.setModifyTime(new Date());
        String json = gson.toJson(report, Report.class);
        client.prepareIndex()
                .setIndex("lawfullyadministration.report")
                .setType(TypeConstant.TYPE)
                .setId(report.getId())
                .setSource(json)
                .execute()
                .actionGet();
        logger.info(report.getId() + "入库78完成");
        logger.info("当前入库第"+i+"条记录");
    }
}