目录
第一种：直接取分页数据
/**
* 处理内存分页
* @param list
* @param pageNo
* @param pageSize
* @return
*/
private <T> List<T> getPageConfigList(List<T> list, int pageNo, int pageSize){
List<T> currentPagelist = null;
if (list != null && !list.isEmpty()) {
currentPagelist = new ArrayList<T>();
int iEnd = 0;
int iStart = 0;
iStart = pageSize * (pageNo - 1);
iEnd = iStart + pageSize;
if (iEnd > list.size()) {
iEnd = list.size();
}
for (int i = iStart; i < iEnd; i++) {
currentPagelist.add(list.get(i));
}
}
return currentPagelist;
}第二种：先分页，再取分页数据
com.google.common.collect.List实现
CollectionContentUtils.sortList(sList, new String[] {"rtn_urlpathmethod"}, "desc");
List<List<TipConfig>> partition = Lists.partition(sList, length);
page.setData(partition.get((start+length-1)/length));扩展知识
guava使用Lists.partition,Lists.transform小结
有时候我们会遇到分割List,把list分成几份，或者把list的元素转换成另一个类型的元素，使用 guava的Lists.partition,Lists.transform可以帮忙我们更加简单的实现此功能
import com.google.common.base.Function;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.List;
public class Main {
public static void main(String[] args) throws Exception {
List<Long> list = new ArrayList<>();
list.add(1L);
list.add(2L);
list.add(3L);
list.add(4L);
list.add(5L);
list.add(6L);
list.add(7L);
list.add(8L);
list.add(9L);
List<List<Long>> originalPageList = Lists.partition(list, 3);
List<String> pageList = Lists.transform(originalPageList, new Function<List<Long>, String>() {
@Override
public String apply(List<Long> list) {
final StringBuffer pageSkuIds = new StringBuffer();
for(Long info : list) {
pageSkuIds.append("AA_").append(info).append(",");
}
return pageSkuIds.toString();
}
});
System.out.println(pageList.toString());
}
}第三种：遍历的同时，进行分页
一次遍历，边分页边批量插入数据库 ；适合海量数据分批处理。
@DataSource(value = "master")
public void recordRanking() {
logger.error("recordRanking start====================================");
String distributedLockKey = "";//分布式锁key
String rankingSizeKey = "";//某个活动的排行榜大小key
String batchInsertErrorKey = "";//某个活动批次插入报错key
String promotionAndRuleKey = "";//时光机活动规则key
try {
// 获取最近一条需要发放大奖的活动（活动结束即可）
promotionAndRuleKey = "promotionAndRule_promotionType_" + SnsPromotionType.TIMEMACHINE_GAME.getCode();
SnsPromotionShareRuleVo spsrVo = (SnsPromotionShareRuleVo) CacheUtil.getObject(promotionAndRuleKey);
if (spsrVo == null) {
spsrVo = snsPromotionDAO.selectPrixPromotionAndRuleByType(SnsPromotionType.TIMEMACHINE_GAME.getCode());
CacheUtil.setObject(promotionAndRuleKey, spsrVo, 30);
if (spsrVo == null) {
logger.error("没有已结束需要发放大奖的时光机活动");
return;
}
}
Long promotionId = spsrVo.getPromotionId();
distributedLockKey = String.format("distributedLockKey_recordRanking_promotionId_%d", promotionId);
rankingSizeKey = String.format("rankingSizeKey_recordRanking_promotionId_%d", promotionId);
batchInsertErrorKey = String.format("batchInsertErrorKey_recordRanking_promotionId_%d", promotionId);
// 分布式锁（保证下面逻辑不会在同一时间重复执行）
if(!CacheUtil.set(distributedLockKey, distributedLockKey, 60*15)){
Object[] args = {distributedLockKey, promotionId};
logger.error("未获取到分布式锁,key:{},活动id:{}", args);
return;
}
// 先查看排行榜表中是否已经全部写入本次活动的数据
SnsShareRecordRankingExample example = new SnsShareRecordRankingExample();
example.createCriteria().andPromotionIdEqualTo(promotionId);
int count = snsShareRecordRankingDAO.countByExample(example);
String rankingSizeStr = CacheUtil.get(rankingSizeKey);
if (StringUtils.isNotBlank(rankingSizeStr) && Integer.valueOf(rankingSizeStr) <= count) {
logger.error("活动id：" + promotionId + "的排行榜数据已经全部写入临时表,排行榜数量：" + count + ",请勿重复操作");
return;
}
// 处理插入失败的数据
boolean handleFlag = handleBatchInsertErrorList(batchInsertErrorKey, promotionId);
if (handleFlag) {
return;
}
// 获取排行榜
StopWatch sw1 = new StopWatch();
sw1.start();
List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId(promotionId);
sw1.stop();
logger.error("snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionId耗时：" + sw1.getTotalTimeMillis() + "ms");
if (CollectionUtils.isEmpty(ssrList)) {
logger.error("该时光机活动排行榜为空,活动id：" + promotionId);
return;
}
int rankingSize = ssrList.size();
// 将排行榜大小放入缓存
CacheUtil.set(rankingSizeKey, String.valueOf(rankingSize));
logger.error("活动id：" + promotionId + "，排行榜数量：" + rankingSize);
// 分批次写入
StopWatch sw2 = new StopWatch();
sw2.start();
// n:计数器,m:页数,l:模数,maxSize:每批处理数量
int n = 0, m = 0, size = rankingSize, maxSize = 500, l = size%maxSize;
int flag = (l == 0) ? 0 : 1;
size = size/maxSize;
List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(maxSize);
SnsShareRecordRanking ssrrTemp = null;
for (int i = 0; i < rankingSize; i++) {
SnsShareRecord ssr = ssrList.get(i);
ssrrTemp = new SnsShareRecordRanking();
ssrrTemp.setShareRecordId(ssr.getId());
ssrrTemp.setPromotionId(ssr.getPromotionId());
ssrrTemp.setUserId(ssr.getUserId());
ssrrTemp.setPrixResultState(ssr.getPrixResultState());
ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
ssrrTemp.setRanking(i+1);
ssrrListTemp.add(ssrrTemp);
n++;
//预处理数据不能整除最大行数时,最后几行数据的处理,n==l表示最后一部分数据;
if(size == m && n == l) {
flag = 0;
}
/**
* case1：大于等于最大行数
* case2：预处理数据不能整除最大行数时,最后几行数据的处理,flag=0
* case3：预处理数据量可以整除最大行数,flag=0
*/
if(n >= maxSize || flag == 0) {
try {
snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
Object[] args = {m, ssrrListTemp.size(), rankingSize};
logger.error("snsShareRecordRankingDAO.batchInsert 第{}次写入成功,本次写入数据量：{},总数据量：{}", args);
} catch (Exception e) {
Object[] args = {m, ssrrListTemp.size(), rankingSize};
logger.error("snsShareRecordRankingDAO.batchInsert 第{}次写入失败,本次写入数据量：{},总数据量：{}", args);
logger.error("snsShareRecordRankingDAO.batchInsert has error", e.getMessage());
// 记录该批次数据到缓存,后面重新插入
String field = String.format("promotionId_%d_page_%d", promotionId, m);
Integer rankingMix = ssrrListTemp.get(0).getRanking();
Integer rankingMax = ssrrListTemp.get(ssrrListTemp.size()-1).getRanking();
String range = String.format("%d_%d", rankingMix, rankingMax);
CacheUtil.hset(batchInsertErrorKey, field, range);
} finally {
ssrrListTemp.clear();
n = 0;
m++;
}
}
}
sw2.stop();
logger.error("分批次写入排行榜数据总耗时：" + sw2.getTotalTimeMillis() + "ms");
} catch (Exception e) {
logger.error("recordRanking has error", e.getMessage());
} finally {
// 释放分布式锁
CacheUtil.del(distributedLockKey);
logger.error("recordRanking end====================================");
}
}
/**
* 处理批次插入失败的数据
* @param batchInsertErrorKey
* @param promotionId
* @return
*/
private boolean handleBatchInsertErrorList(String batchInsertErrorKey, Long promotionId) {
logger.error("handleBatchInsertErrorList start==========batchInsertErrorKey：" + batchInsertErrorKey);
boolean flag = false;
StopWatch sw1 = new StopWatch();
sw1.start();
Map<String, String> batchInsertErrorMap = CacheUtil.hgetAll(batchInsertErrorKey);
if (MapUtils.isNotEmpty(batchInsertErrorMap)) {
// 将批量插入失败的数据重新插入一次
for (Map.Entry<String, String> entry : batchInsertErrorMap.entrySet()) {
String field = entry.getKey();
String range = entry.getValue();
String[] rangeArr = StringUtils.split(range, "_");
Integer rankingMix = Integer.valueOf(StringUtils.replace(rangeArr[0], "\"", ""));
Integer rankingMax = Integer.valueOf(StringUtils.replace(rangeArr[1], "\"", ""));
// 获取排名范围内的排行榜数据
List<SnsShareRecord> ssrList = snsShareRecordDAO.getListOfAccomplishedAndFilteredByPromotionIdAndRange(promotionId, rankingMix-1, rankingMax - rankingMix + 1);
if (CollectionUtils.isEmpty(ssrList)) {
Object[] args1 = {field};
logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}没有获取到该范围排行榜数据,下面将该field从缓存删除", args1);
CacheUtil.hdel(batchInsertErrorKey, field);
continue;
}
int size = ssrList.size();
Object[] args2 = {field, size};
logger.error("handleBatchInsertErrorList batchInsertErrorMap中field:{}对应的数据量:{},下面将进行批量插入", args2);
List<SnsShareRecordRanking> ssrrListTemp = new ArrayList<>(size);
SnsShareRecordRanking ssrrTemp = null;
int ranking = rankingMix.intValue();//最小排名
for (int i = 0; i < size; i++) {
SnsShareRecord ssr = ssrList.get(i);
ssrrTemp = new SnsShareRecordRanking();
ssrrTemp.setShareRecordId(ssr.getId());
ssrrTemp.setPromotionId(ssr.getPromotionId());
ssrrTemp.setUserId(ssr.getUserId());
ssrrTemp.setPrixResultState(ssr.getPrixResultState());
ssrrTemp.setTotalRegisterDuration(ssr.getTotalRegisterDuration());
ssrrTemp.setRanking(ranking + i);
ssrrListTemp.add(ssrrTemp);
}
// 重新执行批量插入
try {
snsShareRecordRankingDAO.batchInsert(ssrrListTemp);
// 插入成功之后删除该批次失败标记
CacheUtil.hdel(batchInsertErrorKey, field);
Object[] args3 = {size};
logger.error("handleBatchInsertErrorList.batchInsert 写入成功,本次写入数据量：{}", args3);
} catch (Exception e) {
Object[] args4 = {size};
logger.error("handleBatchInsertErrorList.batchInsert 写入失败,本次写入数据量：{}", args4);
logger.error("handleBatchInsertErrorList.batchInsert has error", e.getMessage());
} finally {
ssrrListTemp.clear();
}
}
flag = true;
}
sw1.stop();
logger.error("handleBatchInsertErrorList end==========batchInsertErrorKey：" + batchInsertErrorKey + ",flag is " + flag + ",耗时：" + sw1.getTotalTimeMillis() + "ms");
return flag;
}
398
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
