java dubbo服务导出excel数据量过大解决方案--2(bug修复)

新问题

• 1.如果调用删除失败后没有补偿机制，这样会导致orderService服务端的内存会越来越少，最终会内存溢出
• 2.我们的orderService服务是会集群部署的，但是我没有配置hash一致性负载均衡策略，这样取分批拿数据的时候会有大问题，拿到的数据不全，有可能会出现拿不到数据的情况，这是个大BUG

真的上面的2个BUG，代码做了修改，上代码只贴有改动的

• admin dubbo配置文件

<!-- 订单导出服务类配置，使用一致性hash负载均衡策略，默认第一个参数的一致性hash，相同会被分发到通一个服务提供者 -->
    <dubbo:reference interface="com.summersoft.ts.order.services.OrderExportService" >
        <dubbo:method name="orderExportMethods"  loadbalance="consistenthash" timeout="1200000"/>
    </dubbo:reference>

• orderService dubbo配置文件

<!-- 订单导出服务类配置，使用一致性hash负载均衡策略，默认第一个参数的一致性hash，相同会被分发到通一个服务提供者 -->
    <dubbo:service interface="com.summersoft.ts.order.services.OrderExportService" ref="orderExportServiceImpl">
        <dubbo:method name="orderExportMethods" loadbalance="consistenthash" timeout="600000"/>
    </dubbo:service>

• controller层代码

public void export(HttpServletRequest request, HttpServletResponse response) {
        // 文件名获取
        Map<String, Object> paramsMap = getMap(request);
        int typeModule = Integer.parseInt ((String) paramsMap.get("typeModule")) ;
        String typeModuleName = "";
        if (typeModule == Constants.BUSINESS_TYPE_TAXI) {
            typeModuleName = "出租车";
        } else if (typeModule == Constants.BUSINESS_TYPE_SPEC) {
            typeModuleName = "专车";
        } else if (typeModule== Constants.BUSINESS_TYPE_EXPRESS) {
            typeModuleName = "快车";
        }
        String fileName = typeModuleName+"订单" ;
        String startTime = (String) paramsMap.get("startTime");
        String endTime = (String) paramsMap.get("endTime");
        //startTime和endTime有一个为空就不拼接时间
        if (StringUtils.isNotEmpty(startTime) && StringUtils.isNotEmpty(endTime) ) {
            startTime = startTime.replaceAll("-", "");
            endTime = endTime.replaceAll("-", "");
            fileName += startTime + "-" + endTime;
        }
        ExcelUtil.setResponseHeader(response,fileName);
        OutputStream out = null;
        BufferedOutputStream bos = null;
        String key = UUID.randomUUID().toString()+"-"+System.currentTimeMillis();
        AjaxList ajaxList;
        try {
            out = response.getOutputStream();
            bos = new BufferedOutputStream(out);
            ajaxList= orderExportService.orderExportMethods(key,paramsMap,0,0,0);
            if (ajaxList.isSuccess()) {
                int count = (int)ajaxList.getData();
                String path = request.getSession().getServletContext().getRealPath("/");
                String excelPath = path + "excel/";
                File excelFile = new File(excelPath);
                if (!excelFile.exists()) {
                    excelFile.mkdir();
                }
                // 压缩文件
                File zip = new File(path + "excel/"  + fileName + ".zip");
                //创建excel
                Workbook book = new HSSFWorkbook();
                String file = path + "excel/" + fileName + ".xls";
                FileOutputStream fos = new FileOutputStream(file);
                Sheet sheet = null;
                for (int i = 0; i <= count; i+=10000) {
                    AjaxList ajaxList1 = orderExportService.orderExportMethods(key, null,i, i + 10000,1);
                    if (ajaxList1.isSuccess()) {
                        sheet = book.createSheet((i+1)+"-"+(i+10000));
                        List<GeneralOrderBean> list= (List<GeneralOrderBean>)ajaxList1.getData();
                        toExcel(list,book,sheet,fos);
                    }
                }
                book.write(fos);
                fos.flush();
                fos.close();
                //删除
                orderExportService.orderExportMethods(key, null, 0, 0, 2);

                File []srcfile = new File[1];
                srcfile[0] = new File(file);

                ExcelUtil.zipFiles(srcfile, zip);
                FileInputStream inStream = new FileInputStream(zip);
                BufferedInputStream bif = new BufferedInputStream(inStream);
                byte[] buf = new byte[10240];
                int readLength;
                while ((readLength = bif.read(buf)) != -1) {
                    bos.write(buf, 0, readLength);
                }
                bif.close();
                List<String> fileNames = new ArrayList<>();
                fileNames.add(file);
                ExcelUtil.deleteFile(fileNames, path + "excel/"  + fileName + ".zip");
            }
        } catch (IOException e1) {
            e1.printStackTrace();
        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            try {
                bos.flush();
                bos.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }

• orderService 服务端接口

public interface OrderExportService {

    /**
     * 后台-订单-通用订单列表查询
     *
     * @param params 参数
     * @param key    缓存key
     * @param type   方法作用类型 0-查询总订单量，1-分段从缓存获取数据，2-删除缓存
     * @return List
     */
    AjaxList orderExportMethods(String key, Map<String, Object> params, int start, int end, int type);
}

• orderService 服务端接口-实现

@Service
public class OrderExportServiceImpl implements OrderExportService {

    @Autowired
    private OrderMapper orderMapper;

    @Override
    public AjaxList orderExportMethods(String key, Map<String, Object> params, int start, int end, int type) {
        switch (type) {
            case 0:
            return listExtend(params, key);
            case 1:
                return getOrderListFromCache(key, start, end);
            case 2:
                return deleteOrderListFromCache(key);
            default:
                return AjaxList.createError("位置方法类型");
        }
    }

    /**
     *  根据条件查询订单总数据，放入本地缓存
     * @param params
     * @param key
     * @return
     */
    private AjaxList listExtend(Map<String, Object> params,String key) {
        List<GeneralOrderBean> generalOrderBeanList=orderMapper.listExtend(params);
        if (CollectionUtils.isEmpty(generalOrderBeanList)) {
            return AjaxList.createError("获取订单信息失败");
        }
        OrderExportCacheUtil.ordereXportCacheMap.put(key, generalOrderBeanList);
        return  AjaxList.createSuccess("获取订单信息成功",generalOrderBeanList.size());
    }

    /**
     *  分段获取本地缓存订单数据
     * @param key
     * @param start
     * @param end
     * @return
     */
    private AjaxList getOrderListFromCache(String key, int start, int end) {
        List<GeneralOrderBean> generalOrderBeanList = OrderExportCacheUtil.getGeneralOrderBeanList(key, start, end);
        if (CollectionUtils.isEmpty(generalOrderBeanList)) {
            return AjaxList.createError("获取失败");
        }
        return AjaxList.createSuccess("获取成功", generalOrderBeanList);
    }

    /**
     *  删除本地缓存数据
     * @param key
     * @return
     */
    private AjaxList deleteOrderListFromCache(String key) {
        OrderExportCacheUtil.delectListByKey(key);
        return AjaxList.createSuccess("删除成功");
    }
}

• 缓存工具类 （主要是删除方法有改动，增加删除失败补偿机制）

public class OrderExportCacheUtil {
    /**
     *  用来缓存
     */
    public static Map<String, List<GeneralOrderBean>> ordereXportCacheMap = new HashMap<>();
    /**
     * 分段获取generalOrderBeanList
     * @param key
     * @param start 包括
     * @param end 不包括
     * @return
     */
    public static List<GeneralOrderBean> getGeneralOrderBeanList(String key ,int start,int end) {
        List<GeneralOrderBean> generalOrderBeanList = null;
        if (ordereXportCacheMap.containsKey(key)) {
           generalOrderBeanList = ordereXportCacheMap.get(key);
            if (!CollectionUtils.isEmpty(generalOrderBeanList) && generalOrderBeanList.size() > start && end > start) {
                if (generalOrderBeanList.size() >= end) {
                    return generalOrderBeanList.subList(start, end);
                } else if (generalOrderBeanList.size() < end) {
                    return generalOrderBeanList.subList(start, generalOrderBeanList.size());
                }
            }
        }
        return null;
    }

    public static void delectListByKey(String key) {
        if (ordereXportCacheMap.containsKey(key)) {
            ordereXportCacheMap.remove(key);
        }
        for (Map.Entry<String, List<GeneralOrderBean>> entry : ordereXportCacheMap.entrySet()) {
            System.out.println("Key = " + entry.getKey() + ", Value = " + entry.getValue());
            String entryKey = entry.getKey();
            String entryKeyTime = entryKey.split("-")[1];
            //清除超过半个小时的缓存数据，删除失败的补偿机制
            if (System.currentTimeMillis() - (Long.parseLong(entryKeyTime)) >= 30 * 60 * 1000) {
                ordereXportCacheMap.remove(entryKey);
            }
        }
    }
}

总结

写代码想的还是不够周全，这么严重的bug出现，的好好反思下，这次印象非常深刻了。