比较两张100W条数据的表

/**
 * @author corpse
 * @since 2020.12.14
 */
public class CheckUpKeep extends AbsTestService
{
    private class Grouper
    {
        private Grouper() throws FileNotFoundException {}
        private final int LIMIT = 1000;

        Set<String> codes = new HashSet<>();

        FileOutputStream out = new FileOutputStream("D:\\out\\ins\\out.txt");

        Map<String, UpkeepWorkBean> map = new HashMap<>();

        int progress = 0;

        int count = 0;

        void push(UpkeepWorkBean work)
        {
            if (codes.size() == LIMIT) {
                deal();

                map.clear();
                codes.clear();
            }

            this.codes.add(work.getCarCode());
            map.put(work.getWorkCode(), work);
        }

        private final String FOMMAT = "id: {0}, work code: {1}, car code: {2}\n";

        void deal()
        {
            if (!codes.isEmpty()) {
                this.progress += LIMIT;
                XhLogger.debug("进度: {0}", progress);

                XhHQL hql = XhHQL.from(UpkeepBean.class)
                                 .in("carCode", InHelper.in(codes));
                List<UpkeepBean> list = find(hql);
                Set<String> set = XhCollection.toSet(list,
                                                     UpkeepBean::getEasWorkNum);

                set.forEach(map::remove);

                if (!map.isEmpty()) {
                    count += map.size();

                    map.values().forEach(work -> {
                        try {
                            out.write(XhCodec.toBytes(XhString.format(FOMMAT,
                                                                      work.getId(),
                                                                      work.getWorkCode(),
                                                                      work.getCarCode())));
                            out.flush();
                        }
                        catch (IOException e) {
                        }
                    });
                }
            }
        }
    }

    @Test
    public void checkUpKeep()
    {
        XhHQL hql = XhHQL.from(UpkeepWorkBean.class)
                         .eq("basicWorkStatus", BasicWorkStatus.overReport)
                         .eq("carCode", "121313")
                         .asc("carCode");
        stream(hql, stream -> {
            try {
                Grouper grouper = new Grouper();
                stream.forEach(bean -> grouper.push((UpkeepWorkBean) bean));
                grouper.deal();

                System.out.println(XhString.format("数量: {0}", grouper.count));
            }
            catch (FileNotFoundException e) {
            }

            return null;
        });
    }
}

主要的思想就是以流的形式循环,数据较多时采用分组的方式逐段比较。技术背景为hibernate,公司架构师封装的一些轮子。业务背景表A(报案表)新增一条,表B(施工表)根据车架号修改状态。但是两张表的数据大概有100万左右,普通方法比较几乎可能,公司大牛随手写了一个小东西,记录一下

  • 0
    点赞
  • 3
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值