这几天在做一个项目,涉及到大批量处理xml报文回执的操作,一开始用的单线程,结果处理了10000多个文件,用了半个多小时还没处理完,后来就考虑多线程,结果发现老发生文件解析冲突,之后就用到了线程队列的东东....
关键代码如下:
package XXXXXX.web.sd.receipt;
import java.io.File;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import jcifs.smb.SmbFile;
import jcifs.smb.SmbFileInputStream;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import com.routdata.gzdatachage.logic.common.ILoadDataLogic;
import com.routdata.gzdatachage.util.CommonUtils;
import com.routdata.gzdatachage.util.Const;
/**
* 报关单回执信息入库
* @author
* @version Oct 18, 2015
*/
//程序线程入口类
public class DeclareSingleGjReceiptJob implements Runnable {
private ILoadDataLogic loadDataLogic;
public ILoadDataLogic getLoadDataLogic() {
return loadDataLogic;
}
public void setLoadDataLogic(ILoadDataLogic loadDataLogic) {
this.loadDataLogic = loadDataLogic;
}
private final static Logger LOG = Logger.getLogger(DeclareSingleGjReceiptJob.class);
@Override
public void run() {// 电商企业订单信息申报
try {
while(true){
LOG.info("回执入库任务开始......");
Long downLoadType =Long.parseLong(CommonUtils.findUploadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_TYPE)); //获取下载文件类型(0 本地 1 远程)
if (downLoadType==0) {
getCustThreadsByDir(); //本地目录
}else {
//getShareCustThreadsByDir(); //共享目录
}
LOG.info("回执入库任务结束......");
Thread.sleep(20000);
}
} catch (Exception e) {
e.printStackTrace();
}
}
//获取客户目录分开线程个数
public void getCustThreadsByDir() throws Exception {
//获取客户发送目录路径
String sendDirPath = CommonUtils.findDownloadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_DIR);
LocalFile instance = new LocalFile();
File custSendDir = new File(sendDirPath);
instance.Quentstart(20, loadDataLogic); //初始化20个线程
if (custSendDir != null && custSendDir.isDirectory()) {
if(custSendDir.listFiles() != null && custSendDir.list().length > 0) {
instance.addFilesIgnoreInterrupted(custSendDir.listFiles());
}else {
LOG.info("总署报关单 下载文件路径:" + custSendDir +"目录为空,没有文件可解析");
}
instance.shutdown();
instance.waiting();
}
}
//获取客户目录分开线程个数
public void getShareCustThreadsByDir() throws Exception {
//获取客户发送目录路径
String sendDirPath = "smb:"+CommonUtils.findDownloadPath(Const.FILE_TPYE_HG, Const.DOWNLOAD_DIR);
ShareFile instance = new ShareFile();
SmbFile custSendDir = new SmbFile(sendDirPath);
instance.Quentstart(30, loadDataLogic); //初始化10个线程
if (custSendDir != null && custSendDir.isDirectory()) {
if(custSendDir.listFiles() != null && custSendDir.list().length > 0) {
instance.addFilesIgnoreInterrupted(custSendDir.listFiles());
}else {
LOG.info("总署报关单 下载文件路径:" + custSendDir +"目录为空,没有文件可解析");
}
instance.shutdown();
instance.waiting();
}
}
}
//主要实现类
package com.routdata.gzdatachage.web.sd.receipt;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.log4j.Logger;
import org.dom4j.Document;
import org.dom4j.Element;
import org.dom4j.io.SAXReader;
import com.routdata.gzdatachage.logic.common.ILoadDataLogic;
import com.routdata.gzdatachage.util.CommonUtils;
import com.routdata.gzdatachage.util.Const;
/**
* Class description goes here.
*
* @author Administrator
* @since 2016-4-24
*/
public class LocalFile {
private final static Logger LOG = Logger.getLogger(DeclareSingleGjReceiptJob.class);
private static int i = 0;
public static final int THREAD_POOL_SIZE = 2;
private final File POISON = new File("");
private final BlockingQueue<File> files = new LinkedBlockingQueue<File>(1000);
private final ConcurrentLinkedQueue<String> infos = new ConcurrentLinkedQueue<String>();
private Thread[] pool = null;
private volatile boolean running = false;
public void Quentstart(int poolSize, ILoadDataLogic loadDataLogic) {
//this.custDir = custDir;
//this.compCode = compCode;
this.loadDataLogic = loadDataLogic;
pool = new Thread[poolSize];
FileWorker worker = new FileWorker();
LOG.info("新启动风信子线程:"+poolSize);
for(int i=0;i<poolSize;i++){
pool[i] = new Thread(worker,"线程"+(i+1));
pool[i].start();
LOG.info(pool[i].getName()+"------启动");
}
running = true;
}
private class FileWorker implements Runnable {
@Override
public void run() {
File file = null;
try {
while((file=files.take())!= POISON){
LOG.debug(Thread.currentThread().getName()+"准备取数据...");
//while(true){
LOG.info(Thread.currentThread().getName()+"将数据"+file.getName()+"从队列中取出,目前数据总量为:"+files.size());
try {
doWork(file);
} catch (Exception e) {
onException(e,file);
}
}
files.put(POISON);
} catch (InterruptedException e) {
}
}
private void onException(Exception e, File file) {
e.printStackTrace();
}
private int doWork(File readFile) {
//TODO 你要做的事情
}
}
public void addFilesIgnoreInterrupted(File[] files) {
for(File file : files){
try {
LOG.debug(Thread.currentThread().getName()+"准备放数据!");
this.files.put(file);
LOG.debug(Thread.currentThread().getName()+"已经放了数据!"+file.getName());
} catch (InterruptedException e) {
}
}
}
public void shutdown(){
try {
if(running){
running = false;
files.put(POISON);
}
} catch (Exception e) {
}
}
public void waiting(){
if(running || !files.contains(POISON)){
shutdown();
//throw new IllegalStateException("You must call shutdown() function before.");
}
for(Thread t : pool){
try {
t.join();
} catch (InterruptedException e) {
}
}
}
public Queue<String> getInfos(){
return infos;
}
public static final Logger logger = Logger
.getLogger(LocalFile.class);
private ILoadDataLogic loadDataLogic;
/**
* @return the loadDataLogic
*/
public ILoadDataLogic getLoadDataLogic() {
return loadDataLogic;
}
/**
* @param loadDataLogic the loadDataLogic to set
*/
public void setLoadDataLogic(ILoadDataLogic loadDataLogic) {
this.loadDataLogic = loadDataLogic;
}
private String threadNo;
//private String custDir;//客户目录
// private String compCode;//客户备案号
/*public String getCustDir() {
return custDir;
}
public void setCustDir(String custDir) {
this.custDir = custDir;
}*/
public String getThreadNo() {
return threadNo;
}
public void setThreadNo(String threadNo) {
this.threadNo = threadNo;
}
}
===============================================以上看不懂没关系,下面有个例子,拷过去看一下就明白了 执行main方法===========================================================================
package test;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
public class Quene {
public static void main(String[] args) {
final BlockingQueue queue = new ArrayBlockingQueue(100);
for(int i=0;i<5;i++){
new Thread("线程"+i){
public void run(){
// while(true){
try {
long time=(long)(Math.random()*1000);
Thread.sleep(time);
System.out.println(Thread.currentThread().getName() + "准备放数据!");
queue.put(time);
System.out.println(Thread.currentThread().getName() + "已经放了数据:"+time +
"队列目前有" + queue.size() + "个数据");
} catch (InterruptedException e) {
e.printStackTrace();
}
// }
}
}.start();
}
for(int i=0;i<2;i++){
new Thread("线程"+i){
public void run(){
while(true){
try {
//将此处的睡眠时间分别改为100和1000,观察运行结果
Thread.sleep(200);
System.out.println(Thread.currentThread().getName() + "准备取数据!");
Object ta=queue.take();
System.out.println(Thread.currentThread().getName() + "已经取走数据:"+ta +
"队列目前有" + queue.size() + "个数据");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}.start();
}
}
}