多线程写文件操作

最近项目有个需求，对数据库中的号码进行检查，检查号码状态是否正常，检查方法需要依赖接口，每个号码调一次http接口，

基本思路：

1，轮询分页查询数据库，取号码

2，对取出的号码循环调接口查询

3，接口结果记录到文件中

由于最初没有考虑大量号码的情况，使用的是单线程处理，当时2W号码用了4个小时，简直跑废了的感觉，可能接口返回比较慢接近1s

后边就使用了多线程处理，网上查了下，基本都是多线程处理数据，然后缓存到队列，单独起一个写文件线程，从队列中取数据，然后写入，实际处理10W数据，使用30分钟左右，提升很大

代码利用得队列是ConcurrentLinkedQueue，多线程安全，不需要考虑自己进行同步加锁操作，不过网上说如果像下边这种写法需要自己进行同步

if(!queue.isEmpty()) {    
   queue.poll(obj);    
}  
需要自己进行同步
synchronized(queue) {    
   if(!queue.isEmpty()) {    
   queue.poll(obj);    
  }    
 }   

下边贴代码：

@Service
public class CheckDataService {
    public static Logger LOGGER = LoggerFactory.getLogger(CheckDataService.class);
 
    @Autowired
    ADao aDao;
 
    public JSONObject checkPackageOrder(String merchantId, String orderNumber, String mnoCode) {
        LOGGER.info("startTime :" + DateUtil.getYearMonthDayHourMinuteSecond(System.currentTimeMillis()));
        long totalCount = 0;
//        long failedCount = 0;
//        long successCount = 0;
        totalCount = aDao.getAmount(merchantId, mnoCode, orderNumber);
        //文件名
        String fileName = merchantId + "_" + orderNumber + "_" + System.currentTimeMillis() + "_result";
        //文件地址
        String resultFileName = IotContext.BOSS_FILE_BASE_DIR + "upload/" + fileName;
 
        ThreadPoolExecutor executor = new ThreadPoolExecutor(20, 50, 10, TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(100), new ThreadPoolExecutor.CallerRunsPolicy());
        ConcurrentLinkedQueue concurrentLinkedQueue = new ConcurrentLinkedQueue();
 
        //写入文件线程
        new Thread(new dealThread(concurrentLinkedQueue, resultFileName)).start();
        try {
            int offset = 0, limit = 10000;
            List<String> phoneNumberList = aDao.get(orderNumber, offset, limit);
            LOGGER.info("ThreadPool start:" + System.currentTimeMillis());
            while (null != phoneNumberList && phoneNumberList.size() > 0) {
                LOGGER.info("phoneNumberList size : " + phoneNumberList.size());
                for (String phoneNumber : phoneNumberList) {
                    //文件内容缓存至队列
                    executor.execute(new productThread(concurrentLinkedQueue, phoneNumber, mnoCode));
                }
                offset += limit;
                phoneNumberList = aDao.get(orderNumber, offset, limit);
            }
            executor.shutdown();
            executor.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS);
        } catch (InterruptedException e) {
            e.printStackTrace();
            LOGGER.error(e.toString());
        }
        LOGGER.info("ThreadPool end:" + System.currentTimeMillis());
        //增加结束标志通知文件写入停止轮询
        concurrentLinkedQueue.add("end");
 
        JSONObject resultJson = new JSONObject();
        resultJson.put("rtnCode", 0);
        resultJson.put("rtnMsg", "检查号码套餐订购成功");
        resultJson.put("totalCount", totalCount);
//        resultJson.put("failedCount", failedCount);
//        resultJson.put("successCount", successCount);
 
        LOGGER.info("endTime :" + DateUtil.getYearMonthDayHourMinuteSecond(System.currentTimeMillis()) + "     " + resultJson.toString());
        return resultJson;
    }
 
    /**
     * 调接口查询结果缓存到队列中
     */
    class productThread implements Runnable {
        private ConcurrentLinkedQueue queue;
        private String phoneNumber;
        private String mnoCode;
 
        productThread(ConcurrentLinkedQueue queue, String phoneNumber, String mnoCode) {
            this.queue = queue;
            this.phoneNumber = phoneNumber;
            this.mnoCode = mnoCode;
        }
 
        @Override
        public void run() {
            
            //调接口查询结果,自己组合需要的数据
            String data = xxxService.CheckData(phoneNumber,mnoCode);
            queue.add(data);
        }
    }
 
    /**
     * 轮询队列写入文件
     */
    class dealThread implements Runnable {
        private ConcurrentLinkedQueue queue;
        private String resultFileName;
 
        dealThread(ConcurrentLinkedQueue queue, String resultFileName) {
            this.queue = queue;
            this.resultFileName = resultFileName;
        }
 
        @Override
        public void run() {
            BufferedWriter fileOut = null;
            try {
                fileOut = new BufferedWriter(new FileWriter(resultFileName));
                while (true) {
                    String data = (String) queue.poll();
                    if (data != null) {
                        if (data.equalsIgnoreCase("end")) {
                            break;
                        } else {
                            fileOut.write(data);
                            fileOut.newLine();
                        }
                    }
                }
            } catch (IOException e) {
                e.printStackTrace();
                LOGGER.error(e.toString());
            } finally {
                if (fileOut != null) {
                    try {
                        fileOut.flush();
                        fileOut.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        }
 
    }
}