昨 天做了一个东西,要实现解析txt文件,然后入库的功能。开始试验了一下单线程插入,速度实在太慢了,半个小时才插入了2W多条数据,后来改用Java的 线程池启用了30个线程,并发的执行,插入100W条数据用了一个多小时。后来又对业务层的事务做了一些调整,每1000条insert之后才提交一次, 一共开了20个线程,最后100W条数据入库一共用了14分钟不到,平均一分钟7.1W条左右。 代码如下:
- /**
- * 分析Apache日志的定时任务.每天运行数次.
- *
- * @author <a href="mailto:HL_Qu@hotmail.com">Along</a>
- *
- * @version $Revision$
- *
- * @since 2009-2-9
- */
- public class ApacheLogAnalysisTask {
- /**
- * Logger for this class
- */
- private static final Log logger = LogFactory.getLog(ApacheLogAnalysisTask.class);
-
- //总线程数
- private static final int THREAD_COUNT = 20;
-
- //每个线程插入的日志数
- private static final long LOG_COUNT_PER_THREAD = 1000;
-
- //日志文件的位置
- private static final String LOG_FILE = Property.LOG_FILE_PATH + "formatLog.txt";
-
- private IObjectActionDetailService objectActionDetailService;
-
- public void setObjectActionDetailService(IObjectActionDetailService objectActionDetailService) {
- this.objectActionDetailService = objectActionDetailService;
- }
- public void execute() {
- this.multiAnalysisLog();
- }
-
- private void multiAnalysisLog() {
- ExecutorService exePool = Executors.newFixedThreadPool(THREAD_COUNT);
-
- FileReader fr = null;
- BufferedReader br = null;
- long beginLine = 1;
- long endLine = 0;
- String logFileBack = Property.LOG_FILE_PATH + "/formatLog_" + DateUtil.getSystemCurrentDate() + "_" + System.currentTimeMillis() + ".txt";
-
- try {
- //文件拷贝出来一个新的,并将原先的删除.
- FileUtil.copyfile(LOG_FILE, logFileBack, true);
- FileUtil.deleteFile(LOG_FILE);
- System.out.println(logFileBack);
-
- fr = new FileReader(logFileBack);
- br = new BufferedReader(fr);
-
- while ((br.readLine()) != null){
- endLine++;
-
- //每个线程分配固定的行数
- if((endLine - beginLine + 1) == LOG_COUNT_PER_THREAD) {
- exePool.execute(new AnalysisLogTask(logFileBack, beginLine, endLine));
- beginLine = endLine + 1;
- }
- }
-
- //最后一个线程
- if (endLine > beginLine) {
- exePool.execute(new AnalysisLogTask(logFileBack, beginLine, endLine));
- }
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- } catch (IOException e) {
- e.printStackTrace();
- } finally {
- if (br != null) {
- try {
- br.close();
- br = null;
- } catch (IOException e) {
- if (logger.isErrorEnabled()) {
- logger.error("run()", e);
- }
- e.printStackTrace();
- }
- }
-
- if (fr != null) {
- try {
- fr.close();
- fr = null;
- } catch (IOException e) {
- if (logger.isErrorEnabled()) {
- logger.error("run()", e);
- }
- e.pr