上亿级数据量-JDBC插入数据库

转载：http://blog.csdn.net/jzshmyt/article/details/7255761

 
package com.database;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
/**
 * 将亿级数据量,通过JDBC方式,插入到数据库中.关键技术说明
 *   一、为了避免内存溢出问题,每100万条重新建立一次jdbc连接.
 *   二、将连接属性设置为不自动提交,每5万条批量执行一次提交.
 * @author Administrator
 */
public class Import {
	private static final String JDBC_URL = "jdbc:oracle:thin:@localhost:1521:orcl";
	private static final String JDBC_USER = "ttt";
	private static final String JDBC_PASSWORD = "pw123456";
	private static final String JDBC_SQL = "insert into tb_test (col1,col2) values (?,?)";
	
	public static void main(String[] args) throws Exception {
		Class.forName("oracle.jdbc.driver.OracleDriver");
		exe(100000005);//1亿数据量测试通过
	}
	
	public static void exe(int sl) throws SQLException{
		int SIZE = 1000000;//每批的记录数(建议值100万)
		int page = sl/SIZE;//计算批数
		
		for (int i = 0; i < page; i++) {//批量执行
			execute(i*SIZE,(i+1)*SIZE);
		}
		if(sl%SIZE != 0){
			execute(page*SIZE,sl);
		}
	}
	
	private static void execute(int begin, int end) throws SQLException {
		Connection conn = null;
		PreparedStatement pst= null;
		try {
			conn = DriverManager.getConnection(JDBC_URL, JDBC_USER,JDBC_PASSWORD);
			System.out.println("重新打开连接 :" + begin + " - " + end);
			
			conn.setAutoCommit(false);// 设置不自动提交
			pst = conn.prepareStatement(JDBC_SQL);
			
			int recordNum = 0; // 计数器
			int commit_size = 50000;// 每次提交记录数5万
			for (int i = begin; i < end; i++) {
				recordNum++; // 计数
				pst.setString(1, "n" + i);
				pst.setString(2, "v" + i);
				pst.addBatch();
				if (recordNum % commit_size == 0) {
					pst.executeBatch();
					conn.commit();
					System.out.println("提交:" + i);
					conn.setAutoCommit(false);
					pst = conn.prepareStatement(JDBC_SQL);
				}
			}
			if (recordNum % commit_size != 0) {
				pst.executeBatch();
				conn.commit();
				System.out.println("提交:" + recordNum);
			}
		} catch (SQLException e) {
			e.printStackTrace();
		} finally {
			try {
				if (pst != null) pst.close();
			} catch (SQLException e) {
				e.printStackTrace();
			}finally{
				try{
					if (conn != null) conn.close();
					System.out.println("成功关闭连接！");
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

}