现在公司项目中，都有几个核心业务，处理的数据量相较于其他是巨量的，而在我们公司，当下就与一个问题，在现在百亿交易额的处理中，有一个核心业务，就是每日发息（已符合最新法规），以前的发息任务，要跑将近3，4个小时，而我要为下一个交易量的增长，可能是千亿的交易额处理做技术准备，我们都知道，从数据来看，交易量的增长和数据的增长，有时候不是简单的线性关系，数据极有可能是交易量增长倍数的3-5倍，这就是一个挑战，今天和大家分享，这个挑战，我是如何度过的。
1，方向
大的方向，是我们要首先确定的，在问题处理上分三个层次处理
- 业务处理
- 数据处理
- 结构处理
简单介绍下：
（1）业务处理
在业务上，首先确定一个原则：分析实时性业务的必要性，尽量做到非实时，为处理留时间
对应到业务上就是，发息：不是实时发息，虽然有发息时间，但是统一到流量不大的深夜执行，第二天用户收到利息；投资：采取排队机制，将投资结果非实时反馈给用户，为校验留时间；匹配：先做借贷匹配，做占用，然后线上实时匹配（就是做债权缓存池）
（2）数据处理
在数据上，也有一个原则就是500万，热表操作数据控制在500万内
在冷热数据处理上，用触发器在某个时刻，定时移动冷热数据（根据业务），保证热库中的数据是流畅可用的。
（3）结构处理
还是有个原则：按维度多线程处理，solr作为查询第一数据来源，修改时与数据库同步，查不到再走数据库。
2，技术
今天只介绍数据批量提交与多线程结合的例子：
线程类：
package com.xvshu.service.logic.dayinter;
import org.apache.commons.lang.ArrayUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.sql.DataSource;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.Types;
import java.text.MessageFormat;
import java.util.*;
import java.util.concurrent.Callable;
/**
* Created by Administrator on 2015/11/17.
*/
public<span style="color:#ff0000;background-color: rgb(255, 255, 153);"> class InsertBatchThread implements Callable<Integer> </span>{
private final static Logger LOGGER = LoggerFactory.getLogger(InsertBatchThread.class);
private final static String WST_TITLE = "WST_";
private Integer vdate;
private Integer uid;
private Integer count;
private Integer SUCCESS =-1;
private Integer UNSUCCESS ;
private List<Integer> intErrorUsers=new ArrayList<Integer>();
private PushTaskHandler pushTaskHandler;
public PushTaskHandler getPushTaskHandler() {
if (null == pushTaskHandler) {
synchronized (this) {
if (null == pushTaskHandler) {
pushTaskHandler = SpringContextUtils.getBean("pushTaskHandler");
}
}
}
return pushTaskHandler;
}
private IWstTaskService wstTaskService;
public IWstTaskService getWstTaskService() {
if (null == wstTaskService) {
synchronized (this) {
if (null == wstTaskService) {
wstTaskService = SpringContextUtils.getBean("wstTaskService");
}
}
}
return wstTaskService;
}
private IOperateLogService operateLogService;
public IOperateLogService getOperateLogService() {
if (null == operateLogService) {
synchronized (this) {
if (null == operateLogService) {
operateLogService = SpringContextUtils.getBean("operateLogService");
}
}
}
return operateLogService;
}
private ProCommFundsInfoMapper proCommFundsInfoMapper;
public ProCommFundsInfoMapper getProCommFundsInfoMapper() {
if (null == proCommFundsInfoMapper) {
synchronized (this) {
if (null == proCommFundsInfoMapper) {
proCommFundsInfoMapper = SpringContextUtils.getBean("proCommFundsInfoMapper");
}
}
}
return proCommFundsInfoMapper;
}
private DataSource dataSource;
public DataSource getDataSource() {
if (null == dataSource) {
synchronized (this) {
if (null == dataSource) {
dataSource = SpringContextUtils.getBean("dataSource");
}
}
}
return dataSource;
}
/**
* 无参构造函数
*/
public InsertBatchThread(){
}
/**
* 构造函数
*/
public InsertBatchThread(Integer vdate, Integer uid, Integer count){
this.vdate=vdate;
this.uid=uid;
this.count=count;
this.UNSUCCESS=count;
this.dataSource = getDataSource();
}
/**
* 多线程规定好的方法，如果是继承Thread类则必须实现这个方法
*/
public Integer call() {
//初始化时间，计算整个函数耗时用
long time = 0;
long timestart = System.currentTimeMillis();
//count能为-1否则无法查到数据
if(count.equals(-1)){
LOGGER.error("=call=>--count:[" + count + "]----count uis -1 ---------time:" + DateUtil.getCurrentTime());
return null;
}
Integer pageSize =Constant.WMPS_INTERWRAP_BATCHUSERSIZE;
Connection conn = null;
//记录日志
LOGGER.info("=call=>--count:[" + count + "]-------------begain time:" + DateUtil.getCurrentTime());
int result = -1;
try {
conn= getDataSource().getConnection();
conn.setAutoCommit(false);
try {
//操作
List<Integer> listUsers = new ArrayList<Integer>();
if(uid == null){
listUsers = getProCommFundsInfoMapper().selectUserForInsertBatch(vdate,count * pageSize, pageSize);
}else{
listUsers.add(uid);
}
LOGGER.info("=call=>--["+count+"].[get -- listUsers] size:"+listUsers.size()+"-------------end time:" + DateUtil.getCurrentTime());
LOGGER.info("=call=>["+count+"] beginuid:"+listUsers.get(0)+" enduid:"+listUsers.get(listUsers.size() - 1));
time=System.currentTimeMillis();
List<ProCommFundsInfo> listOnePage = getProCommFundsInfoMapper().selectForInsertBatchByUser(vdate, listUsers.get(0), listUsers.get(listUsers.size() - 1));
LOGGER.info("=call=>--["+count+"].[get -- listOnePage] size:"+listOnePage.size()+"-------------spen time:" + (System.currentTimeMillis()-time));
time=System.currentTimeMillis();
List<ProCommFundsInfo> listOnePageNew = getProCommFundsInfoMapper().selectForInsertBatchByUserNew(vdate, listUsers.get(0), listUsers.get(listUsers.size() - 1));
//给procommanlist赋id值与初始状态
for(int i=0;i<listOnePage.size();i++){
if( listOnePage.get(i) == null ){
listOnePage.remove(i);
i--;
}
if(i>=0){
listOnePage.get(i).setId((int)SequenceHelp.get(SequenceHelp.D_PRO_COMM_FUNDS_INFO));
listOnePage.get(i).setStatus(BigDecimal.ONE);
}
}
//------------------------开始操作数据库：--------begin--------------------------
//批量插入proCommFundsInfo
time=System.currentTimeMillis();
proCommFundsInfoInsertBatch(listOnePage,conn);
LOGGER.info("=call=>--["+count+"].[set -- proCommFundsInfoInsertBatch]size:"+listOnePage.size()+"-------------spen time:" + (System.currentTimeMillis()-time));
//------------------------开始操作数据库：--------end------------------------
//提交事务
conn.commit();
//TODO:单独处理发站内信的问题
try {
//发息成功，发站内信
if(listOnePageDayInner!=null && !listOnePageDayInner.isEmpty()){
time=System.currentTimeMillis();
List<NoticetaskVoForBatch> listNotice = new ArrayList<NoticetaskVoForBatch>();
for (WmpsDayInter oneDayInter : listOnePageDayInner) {
NoticetaskVoForBatch oneNotice = new NoticetaskVoForBatch();
oneNotice.setNoticeNumber(Constant.NOTICE_YLB_HX);
oneNotice.setTitleParams(new Object[]{"恭喜您！您已成功收到口口口口的付息！"});
oneNotice.setUuid(createUUID(oneDayInter));
listNotice.add(oneNotice);
}
getLogicNoticetaskService().batchSendNotice(listNotice);
LOGGER.info("=call=>sendNotice spen time :"+(System.currentTimeMillis()-time));
}
}catch(Exception e){
LOGGER.error("=call=>sendNotice error",e);
String wstDayInterNoticeUuid = "WST_DAY_INTER_NOTICE_"+ UUID.randomUUID().toString();
wstTaskService.objectsToWstTask(listOnePageDayInner,wstDayInterNoticeUuid, WstTaskTypeEnum.WST_INSERT_DAYINTER_NOTICE.getValue(),conn);
//发任务引擎
getPushTaskHandler().handlePushTask(JSON.toJSONString(wstDayInterNoticeUuid), WST_SENDNOTICE_INSERT_DAYINTER_TASK_TYPE,UUID.randomUUID().toString() , null, null);
}
}catch (Exception e){
conn.rollback();
result=UNSUCCESS;
e.printStackTrace();
LOGGER.error("=call=>error--["+count+"]-------------spen time:" + (System.currentTimeMillis()-timestart),e);
}
}catch (Exception e){
e.printStackTrace();
LOGGER.error("=call=>error--["+count+"]-------------spen time:" + (System.currentTimeMillis()-timestart),e);
}finally {
try {
if (conn != null) {
conn.setAutoCommit(true);
conn.close();
}
}catch (Exception e){
LOGGER.error("=call=>error close conn",e);
}
}
LOGGER.info("<=call=>--[" + count + "]-------------spen time:" + (System.currentTimeMillis() - timestart));
return result;
}
/**
* 批量插入procomm
* @param listOnePage
* @param conn
* @return
* @throws Exception
*/
public int[] proCommFundsInfoInsertBatch(List<ProCommFundsInfo> listOnePage ,Connection conn) throws Exception{
<span style="color:#cc0000;background-color: rgb(255, 255, 153);">int[] results = new int[0];
PreparedStatement ps = conn.prepareStatement("INSERT ignore INTO d_口口口口口口info" +
"(uid,income,rid) values" +
"(?,?,?)");
try {
for (int i =0 ;i< listOnePage.size();i++) {
ProCommFundsInfo oneObj = listOnePage.get(i);
if (oneObj.getUid() == null) {
ps.setNull(1, Types.INTEGER);
} else {
ps.setInt(1, oneObj.getUid());
}
ps.setBigDecimal(2, oneObj.getIncome());
if (oneObj.getRid() == null) {
ps.setNull(3, Types.INTEGER);
} else {
ps.setInt(3, oneObj.getRid());
}
ps.addBatch();
}
int[] result = ps.executeBatch();
results = ArrayUtils.addAll(results,result);
}finally {
ps.close();
}
return results;</span>
}
}
前台控制类
private Boolean insertDayInter(Integer vdate, Integer uid ) {
if(isPushTask){
LOGGER.info("=insertDayInter=> is PushTaskEngine call This Method ");
}
//记录日志
OperateLog operateLog = operateLogService.selectLogForCheckInter(vdate);
//判断是否插入过日志
if(operateLog !=null){
LOGGER.info("=insertDayInter=> operateLog is not null ");
//标记成功，直接返回
if(operateLog.getStatus()!=null && operateLog.getStatus().equals(Constant.OPERATELOG_STATUS_SUCEESS)){
return true;
}else if (operateLog.getStatus()!=null && operateLog.getStatus().equals(Constant.OPERATELOG_STATUS_PROCESSING)){
//时间间隔小于1小时则直接返回
Long time = DateUtil.getNowDate().getTime() - operateLog.getModified().getTime();
if(time<3600000){
return true;
}
}
operateLog.setStatus(Constant.OPERATELOG_STATUS_PROCESSING);
operateLog.setModified(new Date());
operateLogService.updateByPrimaryKeySelective(operateLog);
}else{
LOGGER.info("=insertDayInter=> operateLog is null ");
operateLog = new OperateLog();
operateLog.setId((Long) SequenceHelp.get(SequenceHelp.OPERATE_LOG));
if(uid==null){
operateLog.setLogType(Constant.OPERATELOG_LOGTYPE_INTERWRAP);
}else{
operateLog.setLogType(Constant.OPERATELOG_LOGTYPE_TEST_INTERWRAP);
operateLog.setRemark("uid:"+uid);
}
operateLog.setStatus(Constant.OPERATELOG_STATUS_UNTREATED);
operateLog.setPri(Constant.OPERATELOG_PRI_ONE);
operateLog.setAutoRedo(Constant.OPERATELOG_AUTOREDO_TRUE);
operateLog.setCreated(new Date(vdate*1000L));
operateLog.setModified(new Date());
operateLogService.insertForDayInter(operateLog);
}
if (vdate == null || vdate.intValue() <= 0 ) {
LOGGER.error("=insertDayInter=>error,vdate=" + vdate + ",uid=" + uid);
} else {
int totalNum = 0;
if(uid ==null){
totalNum = proCommFundsInfoService.countForInsertBatchByUser(vdate);
}else{
totalNum=1;
}
if(totalNum==0){
LOGGER.error("=insertDayInter=>error,vdate=" + vdate + ",uid=" + uid+" totalNum is 0 ");
return true;
}
int startRow = 0;
List<FutureTask<Integer>> listFutures = new ArrayList<FutureTask<Integer>>();
//线程发息，每个线程处理n个用户
getTaskExecutor();
try {
LOGGER.info("=insertDayInter=> insert proCommFundsInfo begin");
<span style="color:#ff0000;background-color: rgb(255, 255, 153);">for (int i = 0; i <= Math.floor(totalNum / Constant.口口_口口_口口); i++) {
FutureTask<Integer> futureTask = new FutureTask<Integer>(new InsertBatchThread(vdate, uid, i));
taskExecutor.execute(futureTask);
listFutures.add(futureTask);
}</span>
LOGGER.info("fixedThreadPool is ok");
if(listFutures !=null && !listFutures.isEmpty()){
for(FutureTask<Integer> ontFuture : listFutures){
Integer threadReturn = ontFuture.get();
if(threadReturn ==null ){
LOGGER.error("FutureTask is null "+listFutures.indexOf(ontFuture));
listException.add(listFutures.indexOf(ontFuture));
}else {
if (!threadReturn.equals(-1)) {
listException.add(threadReturn);
}
}
}
}
result =true;
operateLog.setStatus(Constant.OPERATELOG_STATUS_SUCEESS);
operateLog.setLogData("SUCEESS");
operateLog.setModified(new Date());
operateLogService.updateByPrimaryKey(operateLog);
LOGGER.info("=insertDayInter=> insert proCommFundsInfo end");
}catch (Exception e){
operateLog.setStatus(Constant.OPERATELOG_STATUS_FAILURE);
result = false;
if(listException != null && !listException.isEmpty() ){
operateLog.setLogData(com.el.wst.soa.common.utils.StringUtils.ListToString(listException));
}else{
operateLog.setLogData("SUCEESS");
}
operateLog.setModified(new Date());
operateLogService.updateByPrimaryKey(operateLog);
LOGGER.error("<=insertDayInter=>error",e);
e.printStackTrace();
}
}
LOGGER.info("<=insertDayInter=>");
return result;
}spring线程池：
<!-- 线程池 -->
<bean id="taskExecutor"
class="org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor">
<property name="corePoolSize" value="20" />
<property name="maxPoolSize" value="50" />
<property name="queueCapacity" value="500" />
<property name="keepAliveSeconds" value="300" />
<property name="rejectedExecutionHandler">
<bean class="java.util.concurrent.ThreadPoolExecutor$DiscardOldestPolicy" />
</property>
</bean>3，结果
综上所做，可以讲百亿交易量的处理，控制在20分钟内完成（测试10万用户，5个产品，5分钟完成），测试环境：双核四线程，普通pc
总结
经过这一路的走来，验证了一个道理，万事有利必有弊，这样我们虽然可以提升效率，但是代码可读性，扩展性上，有了瓶颈，如果谁有更好的方案，欢迎交流，交流中，我们一起来探讨更大，更复杂数据的处理。
声明：以下内容为技术分享，涉及公司真实业务部分已经掩盖，代码不可直接执行，请大家注意。
扫一扫
1307
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
