java高向低转化,【转】java实现高性能的数据同步

1 packagecom.dlbank.domain;2 3 importjava.sql.Connection;4 importjava.sql.PreparedStatement;5 importjava.sql.ResultSet;6 importjava.sql.Statement;7 importjava.util.List;8 importjava.util.concurrent.atomic.AtomicLong;9 10 importorg.apache.log4j.Logger;11 12 /**13 *

title: 数据同步类 

14 *

Description: 该类用于将生产核心库数据同步到开发库

15 *@authorTank Zhang16 */17 publicclassCoreDataSyncImplimplementsCoreDataSync {18 19 privateListcoreTBNames;//要同步的核心库表名20 privateConnectionFactory connectionFactory;21 privateLogger log=Logger.getLogger(getClass());22 23 privateAtomicLong currentSynCount=newAtomicLong(0L);//当前已同步的条数24 25 privateintsyncThreadNum;//同步的线程数26 27 @Override28 publicvoidsyncData(intbusinessType)throwsException {29 30 for(String tmpTBName : coreTBNames) {31 log.info("开始同步核心库"+tmpTBName+"表数据");32 //获得核心库连接33 Connection coreConnection=connectionFactory.getDMSConnection(4);34 Statement coreStmt=coreConnection.createStatement();35 //为每个线程分配结果集36 ResultSet coreRs=coreStmt.executeQuery("SELECT count(*) FROM"+tmpTBName);37 coreRs.next();38 //总共处理的数量39 longtotalNum=coreRs.getLong(1);40 //每个线程处理的数量41 longownerRecordNum=(long) Math.ceil((totalNum/syncThreadNum));42 log.info("共需要同步的数据量："+totalNum);43 log.info("同步线程数量："+syncThreadNum);44 log.info("每个线程可处理的数量："+ownerRecordNum);45 //开启五个线程向目标库同步数据46 for(inti=0; icoreTBNames) {72 this.coreTBNames=coreTBNames;73 }74 75 publicvoidsetConnectionFactory(ConnectionFactory connectionFactory) {76 this.connectionFactory=connectionFactory;77 }78 79 publicvoidsetSyncThreadNum(intsyncThreadNum) {80 this.syncThreadNum=syncThreadNum;81 }82 83 //数据同步线程84 finalclassWorkerHandlerimplementsRunnable {85 ResultSet coreRs;86 String queryStr;87 intbusinessType;88 String targetTBName;89 publicWorkerHandler(String queryStr,intbusinessType,String targetTBName) {90 this.queryStr=queryStr;91 this.businessType=businessType;92 this.targetTBName=targetTBName;93 }94 @Override95 publicvoidrun() {96 try{97 //开始同步98 launchSyncData();99 }catch(Exception e){100 log.error(e);101 e.printStackTrace();102 }103 }104 //同步数据方法105 voidlaunchSyncData()throwsException{106 //获得核心库连接107 Connection coreConnection=connectionFactory.getDMSConnection(4);108 Statement coreStmt=coreConnection.createStatement();109 //获得目标库连接110 Connection targetConn=connectionFactory.getDMSConnection(businessType);111 targetConn.setAutoCommit(false);//设置手动提交112 PreparedStatement targetPstmt=targetConn.prepareStatement("INSERT INTO"+targetTBName+"VALUES (?,?,?,?,?)");113 ResultSet coreRs=coreStmt.executeQuery(queryStr);114 log.info(Thread.currentThread().getName()+"'s Query SQL::"+queryStr);115 intbatchCounter=0;//累加的批处理数量116 while(coreRs.next()) {117 targetPstmt.setString(1, coreRs.getString(2));118 targetPstmt.setString(2, coreRs.getString(3));119 targetPstmt.setString(3, coreRs.getString(4));120 targetPstmt.setString(4, coreRs.getString(5));121 targetPstmt.setString(5, coreRs.getString(6));122 targetPstmt.addBatch();123 batchCounter++;124 currentSynCount.incrementAndGet();//递增125 if(batchCounter%10000==0) {//1万条数据一提交126 targetPstmt.executeBatch();127 targetPstmt.clearBatch();128 targetConn.commit();129 }130 }131 //提交剩余的批处理132 targetPstmt.executeBatch();133 targetPstmt.clearBatch();134 targetConn.commit();135 //释放连接136 connectionFactory.release(targetConn, targetPstmt,coreRs);137 }138 }139 }