java多线程 分页_多线程分页处理大批量数据实例

最新推荐文章于 2023-06-07 10:41:00 发布
最新推荐文章于 2023-06-07 10:41:00 发布
阅读量516 收藏
点赞数
文章标签： java多线程 分页
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/weixin_39866087/article/details/114087558
版权

@Overridepublic voidsyncData(String category) {//判断当前es索引是否存在

SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd");

StringBuffer indexName= null;if(category.equals(StatContant.CATEGORY_TYPE_HOUSE)) {//房源

indexName = new StringBuffer("bi_kanyu_dm_housebase_").append(dateFormat.format(new Date())).append("_1");

}else if(category.equals(StatContant.CATEGORY_TYPE_CUSTORMER)) {//客源

indexName = new StringBuffer("bi_kanyu_dm_customer_").append(dateFormat.format(new Date())).append("_1");

}else if(category.equals(StatContant.CATEGORY_TYPE_PERFORMANCE)) {//业绩

indexName = new StringBuffer("bi_kanyu_dm_performance_").append(dateFormat.format(new Date())).append("_1");

}

String indexName1=indexName.toString();

String clusterNames=AppPropUtil.getPropValue(AppPropUtil.Keys.BI_CLUSTER_NAME);boolean indexExists =searchService.isIndexExists(clusterNames, indexName.toString());if (indexExists) {//如果这个索引存在，先干掉

searchService.deleteIndex(clusterNames, indexName.toString());

}//创建当前对应的索引

boolean createIndex =searchService.createIndex(clusterNames, indexName.toString());if (createIndex) {//创建安索引成功

logger.error("创建索引" + indexName + "成功");//创建对应的mapping

boolean createMapping = false;if(category.equals(StatContant.CATEGORY_TYPE_HOUSE)) {//房源

createMapping = searchService.createMapping(clusterNames, indexName.toString(), ESContant.DM_HOUSEBASE_TYPE, FileUtils.getContentByLine("mappings/dm_housebase.txt"));

}else if(category.equals(StatContant.CATEGORY_TYPE_CUSTORMER)) {//客源

createMapping = searchService.createMapping(clusterNames, indexName.toString(), ESContant.DM_CUSTOMER_TYPE, FileUtils.getContentByLine("mappings/dm_customer.txt"));

}else if(category.equals(StatContant.CATEGORY_TYPE_PERFORMANCE)) {//业绩

createMapping = searchService.createMapping(clusterNames, indexName.toString(), ESContant.DM_PERFORMANCE_TYPE, FileUtils.getContentByLine("mappings/dm_performance.txt"));

}if(createMapping) {

logger.error("创建mapping--"+StatContant.CATEGORY_TYPE_MAP.get(category)+"成功");long begin =System.currentTimeMillis();

logger.error("------------------开始同步BI两个月"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据：开始时间" + DateUtil.getDefaultDate() + "-----------------------");int pageSize = 10000;while (true) {int total = biJdbcTemplate.queryForInt("select count(1) from "+StatContant.CATEGORY_TYPE_MAP.get(category)+"_batch dt where dt.processed=1 and dt.status=0 and to_char(dt.batchtime,'yyyyMMdd') = to_char(sysdate,'yyyyMMdd')");if (total == 0) {break;

}

logger.error("查出BI两个月"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据为：" + total + "条");double querySize =total;double totalPage = Math.ceil(querySize /pageSize);

logger.error("共分" + totalPage + "页");final String sql = "select * from (select t.*,rownum as rn from "+StatContant.CATEGORY_TYPE_MAP.get(category)+"_batch t where t.processed=1 and t.status=0 and to_char(t.batchtime,'yyyyMMdd') = to_char(sysdate,'yyyyMMdd') and rownum<=? order by t.batchtime asc) where rn>=?";

ExecutorService es= Executors.newFixedThreadPool(10);for (int i = 0; i < totalPage; i++) {int start = i * pageSize + 1;int end = (i + 1) *pageSize;if (end >total) {

end=total;

}final Object[] params = newObject[]{end, start};

es.submit(newRunnable() {

@Overridepublic voidrun() {

List> objs = new ArrayList>();

List dmperformances = new ArrayList();

List dmCustomers =new ArrayList();

List dmHouseBases = new ArrayList();if(category.equals(StatContant.CATEGORY_TYPE_CUSTORMER)) {//客源

dmCustomers = biJdbcTemplate.query(sql, params, new RowMapper() {

@Overridepublic DmCustomer mapRow(ResultSet rs, int arg1) throwsSQLException {

DmCustomer dmCustomer= newDmCustomer();

String id=JugHelper.generalUUID();

dmCustomer.setId(id);

dmCustomer.setMid(rs.getObject("MID") != null ? rs.getObject("MID").toString() : "");

dmCustomer.setDimorgid(rs.getObject("DIMORGID") != null ? rs.getObject("DIMORGID").toString() : "");

dmCustomer.setOldorgid(rs.getObject("OLDORGID") != null ? rs.getObject("OLDORGID").toString() : "");

dmCustomer.setDimdateid(rs.getObject("DIMDATEID") != null ? rs.getObject("DIMDATEID").toString() : "");

dmCustomer.setStartdate(rs.getObject("STARTDATE") != null ? rs.getObject("STARTDATE").toString() : "");

dmCustomer.setEnddate(rs.getObject("ENDDATE") != null ? rs.getObject("ENDDATE").toString() : "");

dmCustomer.setIdfullpath(rs.getObject("IDFULLPATH") != null ? rs.getObject("IDFULLPATH").toString() : "");

dmCustomer.setType(rs.getObject("TYPE") != null ? rs.getObject("TYPE").toString() : "");

SimpleDateFormat format= new SimpleDateFormat("yyyy-MM-dd");

String batchTime= "";if (rs.getObject("BATCHTIME") != null) {

batchTime= format.format(rs.getDate("BATCHTIME"));

}

dmCustomer.setBatchtime(batchTime);

dmCustomer.setBatchuser(rs.getObject("BATCHUSER") != null ? rs.getObject("BATCHUSER").toString() : "");

dmCustomer.setEfCusts(rs.getObject("EF_CUSTS") != null ? rs.getObject("EF_CUSTS").toString() : "");

dmCustomer.setBuyCusts(rs.getObject("BUY_CUSTS") != null ? rs.getObject("BUY_CUSTS").toString() : "");

dmCustomer.setSecLookCusts(rs.getObject("SEC_LOOK_CUSTS") != null ? rs.getObject("SEC_LOOK_CUSTS").toString() : "");

dmCustomer.setCrossRegionLookGroups(rs.getObject("CROSS_REGION_LOOK_GROUPS") != null ? rs.getObject("CROSS_REGION_LOOK_GROUPS").toString() : "");

dmCustomer.setLookGroups(rs.getObject("LOOK_GROUPS") != null ? rs.getObject("LOOK_GROUPS").toString() : "");

dmCustomer.setCrossZoneOfAreaLookGroups(rs.getObject("CROSS_ZONE_OF_AREA_LOOK_GROUPS") != null ? rs.getObject("CROSS_ZONE_OF_AREA_LOOK_GROUPS").toString() : "");

dmCustomer.setCrossAreaLookGroups(rs.getObject("CROSS_AREA_LOOK_GROUPS") != null ? rs.getObject("CROSS_AREA_LOOK_GROUPS").toString() : "");

dmCustomer.setCrossAreaOneWithMoreLooks(rs.getObject("CROSS_AREA_ONE_WITH_MORE_LOOKS") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE_LOOKS").toString() : "");

dmCustomer.setAddCusts(rs.getObject("ADD_CUSTS") != null ? rs.getObject("ADD_CUSTS").toString() : "");

dmCustomer.setAddCusts7daysLook(rs.getObject("ADD_CUSTS_7DAYS_LOOK") != null ? rs.getObject("ADD_CUSTS_7DAYS_LOOK").toString() : "");

dmCustomer.setAddCustsDeals(rs.getObject("ADD_CUSTS_DEALS") != null ? rs.getObject("ADD_CUSTS_DEALS").toString() : "");

dmCustomer.setNewSalesPurchasesGroup(rs.getObject("NEW_SALES_PURCHASES_GROUP") != null ? rs.getObject("NEW_SALES_PURCHASES_GROUP").toString() : "");

dmCustomer.setSalesWithViews(rs.getObject("SALES_WITH_VIEWS") != null ? rs.getObject("SALES_WITH_VIEWS").toString() : "");

dmCustomer.setNewCustomersSeeMore(rs.getObject("NEW_CUSTOMERS_SEE_MORE") != null ? rs.getObject("NEW_CUSTOMERS_SEE_MORE").toString() : "");

dmCustomer.setNewRentCustomerGroup(rs.getObject("NEW_RENT_CUSTOMER_GROUP") != null ? rs.getObject("NEW_RENT_CUSTOMER_GROUP").toString() : "");

dmCustomer.setNewRentCustomersWithViews(rs.getObject("NEW_RENT_CUSTOMERS_WITH_VIEWS") != null ? rs.getObject("NEW_RENT_CUSTOMERS_WITH_VIEWS").toString() : "");

dmCustomer.setnTransaCustHisVAvgViews(rs.getObject("N_TRANSA_CUST_HIS_V_AVG_VIEWS") != null ? rs.getObject("N_TRANSA_CUST_HIS_V_AVG_VIEWS").toString() : "");

dmCustomer.setnTransaCustHisHAvgViews(rs.getObject("N_TRANSA_CUST_HIS_H_AVG_VIEWS") != null ? rs.getObject("N_TRANSA_CUST_HIS_H_AVG_VIEWS").toString() : "");

dmCustomer.setnTransaCustAvgPeriod(rs.getObject("N_TRANSA_CUST_AVG_PERIOD") != null ? rs.getObject("N_TRANSA_CUST_AVG_PERIOD").toString() : "");

dmCustomer.setDimorgcnt(rs.getObject("DIMORGCNT") != null ? rs.getObject("DIMORGCNT").toString() : "");

dmCustomer.setDimorgallcnt(rs.getObject("DIMORGALLCNT") != null ? rs.getObject("DIMORGALLCNT").toString() : "");

dmCustomer.setPerbusicnt(rs.getObject("PERBUSICNT") != null ? rs.getObject("PERBUSICNT").toString() : "");

dmCustomer.setPernobusicnt(rs.getObject("PERNOBUSICNT") != null ? rs.getObject("PERNOBUSICNT").toString() : "");

dmCustomer.setPidfullpath(rs.getObject("PIDFULLPATH") != null ? rs.getObject("PIDFULLPATH").toString() : "");

dmCustomer.setCrossAreaLookGroupsCnt(rs.getObject("CROSS_AREA_LOOK_GROUPS_CNT") != null ? rs.getObject("CROSS_AREA_LOOK_GROUPS_CNT").toString() : "");

dmCustomer.setShopcnt(rs.getObject("SHOPCNT") != null ? rs.getObject("SHOPCNT").toString() : "");

dmCustomer.setCrossAreaOneWithMoreGroup(rs.getObject("CROSS_AREA_ONE_WITH_MORE_GROUP") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE_GROUP").toString() : "");

dmCustomer.setSecLookRate(rs.getObject("SEC_LOOK_RATE") != null ? rs.getObject("SEC_LOOK_RATE").toString() : "");

dmCustomer.setCrossAreaOneWithMoreRate(rs.getObject("CROSS_AREA_ONE_WITH_MORE_RATE") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE_RATE").toString() : "");

dmCustomer.setCrossZoneOfAreaLookRate(rs.getObject("CROSS_ZONE_OF_AREA_LOOK_RATE") != null ? rs.getObject("CROSS_ZONE_OF_AREA_LOOK_RATE").toString() : "");

dmCustomer.setCrossAreaLookRate(rs.getObject("CROSS_AREA_LOOK_RATE") != null ? rs.getObject("CROSS_AREA_LOOK_RATE").toString() : "");

dmCustomer.setCrossAreaOneWithMore(rs.getObject("CROSS_AREA_ONE_WITH_MORE") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE").toString() : "");

dmCustomer.setAddCustsDealsRate(rs.getObject("ADD_CUSTS_DEALS_RATE") != null ? rs.getObject("ADD_CUSTS_DEALS_RATE").toString() : "");

dmCustomer.setDimorgName(rs.getObject("DIMORG_NAME") != null ? rs.getObject("DIMORG_NAME").toString() : "");

dmCustomer.setPerBusi(rs.getObject("PER_BUSI") != null ? rs.getObject("PER_BUSI").toString() : "");

dmCustomer.setShopcnt1(rs.getObject("SHOPCNT1") != null ? rs.getObject("SHOPCNT1").toString() : "");

dmCustomer.setAreacnt1(rs.getObject("AREACNT1") != null ? rs.getObject("AREACNT1").toString() : "");

dmCustomer.setDictcnt1(rs.getObject("DICTCNT1") != null ? rs.getObject("DICTCNT1").toString() : "");

dmCustomer.setShopcnt2(rs.getObject("SHOPCNT2") != null ? rs.getObject("SHOPCNT2").toString() : "");

dmCustomer.setAreacnt2(rs.getObject("AREACNT2") != null ? rs.getObject("AREACNT2").toString() : "");

dmCustomer.setDictcnt2(rs.getObject("DICTCNT2") != null ? rs.getObject("DICTCNT2").toString() : "");

dmCustomer.setActperbusi2(rs.getObject("ACTPERBUSI2") != null ? rs.getObject("ACTPERBUSI2").toString() : "");

dmCustomer.setnTransaCustPeriod(rs.getObject("N_TRANSA_CUST_PERIOD") != null ? rs.getObject("N_TRANSA_CUST_PERIOD").toString() : "");

dmCustomer.setnTransaCustHisHViews(rs.getObject("N_TRANSA_CUST_HIS_H_VIEWS") != null ? rs.getObject("N_TRANSA_CUST_HIS_H_VIEWS").toString() : "");

dmCustomer.setNidfullpath(rs.getObject("NIDFULLPATH") != null ? rs.getObject("NIDFULLPATH").toString() : "");

dmCustomer.setNpidfullpath(rs.getObject("NPIDFULLPATH") != null ? rs.getObject("NPIDFULLPATH").toString() : "");

dmCustomer.setProcessed(rs.getObject("PROCESSED") != null ? rs.getObject("PROCESSED").toString() : "");

dmCustomer.setCrossZoneOfAreaLooks(rs.getObject("CROSS_ZONE_OF_AREA_LOOKS") != null ? rs.getObject("CROSS_ZONE_OF_AREA_LOOKS").toString() : "");

dmCustomer.setAreaSLookCnt(rs.getObject("AREA_S_LOOK_CNT") != null ? rs.getObject("AREA_S_LOOK_CNT").toString() : "");

dmCustomer.setAreaSLookGroups(rs.getObject("AREA_S_LOOK_GROUPS") != null ? rs.getObject("AREA_S_LOOK_GROUPS").toString() : "");

dmCustomer.setRepeatLookGroups(rs.getObject("REPEAT_LOOK_GROUPS") != null ? rs.getObject("REPEAT_LOOK_GROUPS").toString() : "");

dmCustomer.setRepeatLookRate(rs.getObject("REPEAT_LOOK_RATE") != null ? rs.getObject("REPEAT_LOOK_RATE").toString() : "");

dmCustomer.setStatus("1");returndmCustomer;

}

});for(DmCustomer dmCustomer : dmCustomers) {try{

objs.add(StatUtils.objectToMap(dmCustomer));

}catch(Exception e) {

e.printStackTrace();

}

}

}else if(category.equals(StatContant.CATEGORY_TYPE_HOUSE)) {

dmHouseBases= biJdbcTemplate.query(sql, params, new RowMapper() {

@Overridepublic DmHouseBase mapRow(ResultSet rs, int arg1) throwsSQLException {

DmHouseBase dmHouseBase= newDmHouseBase();

dmHouseBase.setId(rs.getObject("MID") != null ? rs.getObject("MID").toString() : "");

dmHouseBase.setMid(rs.getObject("MID") != null ? rs.getObject("MID").toString() : "");

dmHouseBase.setDimorgid(rs.getObject("DIMORGID") != null ? rs.getObject("DIMORGID").toString() : "");

dmHouseBase.setOldorgid(rs.getObject("OLDORGID") != null ? rs.getObject("OLDORGID").toString() : "");

dmHouseBase.setDimdateid(rs.getObject("DIMDATEID") != null ? rs.getObject("DIMDATEID").toString() : "");

dmHouseBase.setStartdate(rs.getObject("STARTDATE") != null ? rs.getObject("STARTDATE").toString() : "");

dmHouseBase.setEnddate(rs.getObject("ENDDATE") != null ? rs.getObject("ENDDATE").toString() : "");

dmHouseBase.setIdfullpath(rs.getObject("IDFULLPATH") != null ? rs.getObject("IDFULLPATH").toString() : "");

dmHouseBase.setType(rs.getObject("TYPE") != null ? rs.getObject("TYPE").toString() : "");

SimpleDateFormat format= new SimpleDateFormat("yyyy-MM-dd");

String batchTime= "";if (rs.getObject("BATCHTIME") != null) {

batchTime= format.format(rs.getDate("BATCHTIME"));

}

dmHouseBase.setBatchtime(batchTime);

dmHouseBase.setBatchuser(rs.getObject("BATCHUSER") != null ? rs.getObject("BATCHUSER").toString() : "");

dmHouseBase.setForsalecnt(rs.getObject("FORSALECNT") != null ? rs.getObject("FORSALECNT").toString() : "");

dmHouseBase.setHaskeycnt(rs.getObject("HASKEYCNT") != null ? rs.getObject("HASKEYCNT").toString() : "");

dmHouseBase.setAuthorcnt(rs.getObject("AUTHORCNT") != null ? rs.getObject("AUTHORCNT").toString() : "");

dmHouseBase.setGoodpiccnt(rs.getObject("GOODPICCNT") != null ? rs.getObject("GOODPICCNT").toString() : "");

dmHouseBase.setGoodhoucnt(rs.getObject("GOODHOUCNT") != null ? rs.getObject("GOODHOUCNT").toString() : "");

dmHouseBase.setSecLookCusts(rs.getObject("SEC_LOOK_CUSTS") != null ? rs.getObject("SEC_LOOK_CUSTS").toString() : "");

dmHouseBase.setCrossRegionLookGroups(rs.getObject("CROSS_REGION_LOOK_GROUPS") != null ? rs.getObject("CROSS_REGION_LOOK_GROUPS").toString() : "");

dmHouseBase.setLookGroups(rs.getObject("LOOK_GROUPS") != null ? rs.getObject("LOOK_GROUPS").toString() : "");

dmHouseBase.setCrossZoneOfAreaLookGroups(rs.getObject("CROSS_ZONE_OF_AREA_LOOK_GROUPS") != null ? rs.getObject("CROSS_ZONE_OF_AREA_LOOK_GROUPS").toString() : "");

dmHouseBase.setCrossAreaLookGroups(rs.getObject("CROSS_AREA_LOOK_GROUPS") != null ? rs.getObject("CROSS_AREA_LOOK_GROUPS").toString() : "");

dmHouseBase.setCrossAreaOneWithMoreLooks(rs.getObject("CROSS_AREA_ONE_WITH_MORE_LOOKS") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE_LOOKS").toString() : "");

dmHouseBase.setDimorgcnt(rs.getObject("DIMORGCNT") != null ? rs.getObject("DIMORGCNT").toString() : "");

dmHouseBase.setDimorgallcnt(rs.getObject("DIMORGALLCNT") != null ? rs.getObject("DIMORGALLCNT").toString() : "");

dmHouseBase.setAddHouse(rs.getObject("ADD_HOUSE") != null ? rs.getObject("ADD_HOUSE").toString() : "");

dmHouseBase.setAddSalekey(rs.getObject("ADD_SALEKEY") != null ? rs.getObject("ADD_SALEKEY").toString() : "");

dmHouseBase.setAddSaleauthor(rs.getObject("ADD_SALEAUTHOR") != null ? rs.getObject("ADD_SALEAUTHOR").toString() : "");

dmHouseBase.setAddRentauthor(rs.getObject("ADD_RENTAUTHOR") != null ? rs.getObject("ADD_RENTAUTHOR").toString() : "");

dmHouseBase.setAddRentkey(rs.getObject("ADD_RENTKEY") != null ? rs.getObject("ADD_RENTKEY").toString() : "");

dmHouseBase.setAddGoogpicture(rs.getObject("ADD_GOOGPICTURE") != null ? rs.getObject("ADD_GOOGPICTURE").toString() : "");

dmHouseBase.setPerbusicnt(rs.getObject("PERBUSICNT") != null ? rs.getObject("PERBUSICNT").toString() : "");

dmHouseBase.setPernobusicnt(rs.getObject("PERNOBUSICNT") != null ? rs.getObject("PERNOBUSICNT").toString() : "");

dmHouseBase.setAddSevenActioncnt(rs.getObject("ADD_SEVEN_ACTIONCNT") != null ? rs.getObject("ADD_SEVEN_ACTIONCNT").toString() : "");

dmHouseBase.setPidfullpath(rs.getObject("PIDFULLPATH") != null ? rs.getObject("PIDFULLPATH").toString() : "");

dmHouseBase.setAddRent(rs.getObject("ADD_RENT") != null ? rs.getObject("ADD_RENT").toString() : "");

dmHouseBase.setShopcnt(rs.getObject("SHOPCNT") != null ? rs.getObject("SHOPCNT").toString() : "");

dmHouseBase.setForsaleDutyCnt(rs.getObject("FORSALE_DUTY_CNT") != null ? rs.getObject("FORSALE_DUTY_CNT").toString() : "");

dmHouseBase.setLookCnt(rs.getObject("LOOK_CNT") != null ? rs.getObject("LOOK_CNT").toString() : "");

dmHouseBase.setCrossAreaLookCnt(rs.getObject("CROSS_AREA_LOOK_CNT") != null ? rs.getObject("CROSS_AREA_LOOK_CNT").toString() : "");

dmHouseBase.setCrossRegionHouseLooks(rs.getObject("CROSS_REGION_HOUSE_LOOKS") != null ? rs.getObject("CROSS_REGION_HOUSE_LOOKS").toString() : "");

dmHouseBase.setLatelySevenActioncnt(rs.getObject("LATELY_SEVEN_ACTIONCNT") != null ? rs.getObject("LATELY_SEVEN_ACTIONCNT").toString() : "");

dmHouseBase.setForsaleDutyOwerCnt(rs.getObject("FORSALE_DUTY_OWER_CNT") != null ? rs.getObject("FORSALE_DUTY_OWER_CNT").toString() : "");

dmHouseBase.setHaskeyRate(rs.getObject("HASKEY_RATE") != null ? rs.getObject("HASKEY_RATE").toString() : "");

dmHouseBase.setAuthorRate(rs.getObject("AUTHOR_RATE") != null ? rs.getObject("AUTHOR_RATE").toString() : "");

dmHouseBase.setGoodpicRate(rs.getObject("GOODPIC_RATE") != null ? rs.getObject("GOODPIC_RATE").toString() : "");

dmHouseBase.setForsaleDutyRate(rs.getObject("FORSALE_DUTY_RATE") != null ? rs.getObject("FORSALE_DUTY_RATE").toString() : "");

dmHouseBase.setForsalePerAvg(rs.getObject("FORSALE_PER_AVG") != null ? rs.getObject("FORSALE_PER_AVG").toString() : "");

dmHouseBase.setForsaleShopAvg(rs.getObject("FORSALE_SHOP_AVG") != null ? rs.getObject("FORSALE_SHOP_AVG").toString() : "");

dmHouseBase.setCrossAreaLookRate(rs.getObject("CROSS_AREA_LOOK_RATE") != null ? rs.getObject("CROSS_AREA_LOOK_RATE").toString() : "");

dmHouseBase.setCrossAreaOneWithMoreRate(rs.getObject("CROSS_AREA_ONE_WITH_MORE_RATE") != null ? rs.getObject("CROSS_AREA_ONE_WITH_MORE_RATE").toString() : "");

dmHouseBase.setAddSevenActionRate(rs.getObject("ADD_SEVEN_ACTION_RATE") != null ? rs.getObject("ADD_SEVEN_ACTION_RATE").toString() : "");

dmHouseBase.setLatelySevenActionRate(rs.getObject("LATELY_SEVEN_ACTION_RATE") != null ? rs.getObject("LATELY_SEVEN_ACTION_RATE").toString() : "");

dmHouseBase.setLastTwentyPercentAvg(rs.getObject("LAST_TWENTY_PERCENT_AVG") != null ? rs.getObject("LAST_TWENTY_PERCENT_AVG").toString() : "");

dmHouseBase.setDimorgName(rs.getObject("DIMORG_NAME") != null ? rs.getObject("DIMORG_NAME").toString() : "");

dmHouseBase.setPerBusi(rs.getObject("PER_BUSI") != null ? rs.getObject("PER_BUSI").toString() : "");

dmHouseBase.setShopcnt1(rs.getObject("SHOPCNT1") != null ? rs.getObject("SHOPCNT1").toString() : "");

dmHouseBase.setAreacnt1(rs.getObject("AREACNT1") != null ? rs.getObject("AREACNT1").toString() : "");

dmHouseBase.setDictcnt1(rs.getObject("DICTCNT1") != null ? rs.getObject("DICTCNT1").toString() : "");

dmHouseBase.setShopcnt2(rs.getObject("SHOPCNT2") != null ? rs.getObject("SHOPCNT2").toString() : "");

dmHouseBase.setAreacnt2(rs.getObject("AREACNT2") != null ? rs.getObject("AREACNT2").toString() : "");

dmHouseBase.setDictcnt2(rs.getObject("DICTCNT2") != null ? rs.getObject("DICTCNT2").toString() : "");

dmHouseBase.setActperbusi2(rs.getObject("ACTPERBUSI2") != null ? rs.getObject("ACTPERBUSI2").toString() : "");

dmHouseBase.setNidfullpath(rs.getObject("NIDFULLPATH") != null ? rs.getObject("NIDFULLPATH").toString() : "");

dmHouseBase.setNpidfullpath(rs.getObject("NPIDFULLPATH") != null ? rs.getObject("NPIDFULLPATH").toString() : "");

dmHouseBase.setProcessed(rs.getObject("PROCESSED") != null ? rs.getObject("PROCESSED").toString() : "");

dmHouseBase.setCrossAreaLookCnt(rs.getObject("CROSS_AREA_S_LOOK_CNT") != null ? rs.getObject("CROSS_AREA_S_LOOK_CNT").toString() : "");

dmHouseBase.setAreaSLookCnt(rs.getObject("AREA_S_LOOK_CNT") != null ? rs.getObject("AREA_S_LOOK_CNT").toString() : "");

dmHouseBase.setAreaSLookGroups(rs.getObject("AREA_S_LOOK_GROUPS") != null ? rs.getObject("AREA_S_LOOK_GROUPS").toString() : "");

dmHouseBase.setStatus("1");returndmHouseBase;

}

});for(DmHouseBase dmHouseBase : dmHouseBases) {try{

objs.add(StatUtils.objectToMap(dmHouseBase));

}catch(Exception e) {

e.printStackTrace();

}

}

}else if(category.equals(StatContant.CATEGORY_TYPE_PERFORMANCE)) {

dmperformances= biJdbcTemplate.query(sql, params, new RowMapper() {

@Overridepublic DmPerformance mapRow(ResultSet rs, int arg1) throwsSQLException {

DmPerformance dmperformance= newDmPerformance();

String id=JugHelper.generalUUID();

dmperformance.setId(id);

dmperformance.setMid(rs.getObject("MID") != null ? rs.getObject("MID").toString() : "");

dmperformance.setDimorgid(rs.getObject("DIMORGID") != null ? rs.getObject("DIMORGID").toString() : "");

dmperformance.setOldorgid(rs.getObject("OLDORGID") != null ? rs.getObject("OLDORGID").toString() : "");

dmperformance.setDimdateid(rs.getObject("DIMDATEID") != null ? rs.getObject("DIMDATEID").toString() : "");

dmperformance.setStartdate(rs.getObject("STARTDATE") != null ? rs.getObject("STARTDATE").toString() : "");

dmperformance.setEnddate(rs.getObject("ENDDATE") != null ? rs.getObject("ENDDATE").toString() : "");

dmperformance.setIdfullpath(rs.getObject("IDFULLPATH") != null ? rs.getObject("IDFULLPATH").toString() : "");

dmperformance.setType(rs.getObject("TYPE") != null ? rs.getObject("TYPE").toString() : "");

SimpleDateFormat format= new SimpleDateFormat("yyyy-MM-dd");

String batchTime= "";if (rs.getObject("BATCHTIME") != null) {

batchTime= format.format(rs.getDate("BATCHTIME"));

}

dmperformance.setBatchtime(batchTime);

dmperformance.setBatchuser(rs.getObject("BATCHUSER") != null ? rs.getObject("BATCHUSER").toString() : "");

dmperformance.setPerformanceMoney(rs.getObject("PERFORMANCE_MONEY") != null ? rs.getObject("PERFORMANCE_MONEY").toString() : "");

dmperformance.setPerformanceCnt(rs.getObject("PERFORMANCE_CNT") != null ? rs.getObject("PERFORMANCE_CNT").toString() : "");

dmperformance.setPerformanceMoneySArea(rs.getObject("PERFORMANCE_MONEY_S_AREA") != null ? rs.getObject("PERFORMANCE_MONEY_S_AREA").toString() : "");

dmperformance.setPerformanceMoneyCArea(rs.getObject("PERFORMANCE_MONEY_C_AREA") != null ? rs.getObject("PERFORMANCE_MONEY_C_AREA").toString() : "");

dmperformance.setPerformanceMoneyCDictarea(rs.getObject("PERFORMANCE_MONEY_C_DICTAREA") != null ? rs.getObject("PERFORMANCE_MONEY_C_DICTAREA").toString() : "");

dmperformance.setPerformanceHCnt(rs.getObject("PERFORMANCE_H_CNT") != null ? rs.getObject("PERFORMANCE_H_CNT").toString() : "");

dmperformance.setPerformanceCCnt(rs.getObject("PERFORMANCE_C_CNT") != null ? rs.getObject("PERFORMANCE_C_CNT").toString() : "");

dmperformance.setPerformanceHMoney(rs.getObject("PERFORMANCE_H_MONEY") != null ? rs.getObject("PERFORMANCE_H_MONEY").toString() : "");

dmperformance.setPerformanceCMoney(rs.getObject("PERFORMANCE_C_MONEY") != null ? rs.getObject("PERFORMANCE_C_MONEY").toString() : "");

dmperformance.setPerformanceMoneyGroups(rs.getObject("PERFORMANCE_MONEY_GROUPS") != null ? rs.getObject("PERFORMANCE_MONEY_GROUPS").toString() : "");

dmperformance.setPerformanceMoneyOrders(rs.getObject("PERFORMANCE_MONEY_ORDERS") != null ? rs.getObject("PERFORMANCE_MONEY_ORDERS").toString() : "");

dmperformance.setPerformanceMoneyPers(rs.getObject("PERFORMANCE_MONEY_PERS") != null ? rs.getObject("PERFORMANCE_MONEY_PERS").toString() : "");

dmperformance.setSellamount(rs.getObject("SELLAMOUNT") != null ? rs.getObject("SELLAMOUNT").toString() : "");

dmperformance.setPerformanceMoneySer(rs.getObject("PERFORMANCE_MONEY_SER") != null ? rs.getObject("PERFORMANCE_MONEY_SER").toString() : "");

dmperformance.setRentMoney(rs.getObject("RENT_MONEY") != null ? rs.getObject("RENT_MONEY").toString() : "");

dmperformance.setRentHCnt(rs.getObject("RENT_H_CNT") != null ? rs.getObject("RENT_H_CNT").toString() : "");

dmperformance.setRentCCnt(rs.getObject("RENT_C_CNT") != null ? rs.getObject("RENT_C_CNT").toString() : "");

dmperformance.setRentACnt(rs.getObject("RENT_A_CNT") != null ? rs.getObject("RENT_A_CNT").toString() : "");

dmperformance.setRentHMoney(rs.getObject("RENT_H_MONEY") != null ? rs.getObject("RENT_H_MONEY").toString() : "");

dmperformance.setRentCMoney(rs.getObject("RENT_C_MONEY") != null ? rs.getObject("RENT_C_MONEY").toString() : "");

dmperformance.setGuarFee(rs.getObject("GUAR_FEE") != null ? rs.getObject("GUAR_FEE").toString() : "");

dmperformance.setGuarCnt(rs.getObject("GUAR_CNT") != null ? rs.getObject("GUAR_CNT").toString() : "");

dmperformance.setDutycnt(rs.getObject("DUTYCNT") != null ? rs.getObject("DUTYCNT").toString() : "");

dmperformance.setNewhouseMoney(rs.getObject("NEWHOUSE_MONEY") != null ? rs.getObject("NEWHOUSE_MONEY").toString() : "");

dmperformance.setNewhouseCnt(rs.getObject("NEWHOUSE_CNT") != null ? rs.getObject("NEWHOUSE_CNT").toString() : "");

dmperformance.setNewhouseFee(rs.getObject("NEWHOUSE_FEE") != null ? rs.getObject("NEWHOUSE_FEE").toString() : "");

dmperformance.setDimorgcnt(rs.getObject("DIMORGCNT") != null ? rs.getObject("DIMORGCNT").toString() : "");

dmperformance.setDimorgallcnt(rs.getObject("DIMORGALLCNT") != null ? rs.getObject("DIMORGALLCNT").toString() : "");

dmperformance.setPerbusicnt(rs.getObject("PERBUSICNT") != null ? rs.getObject("PERBUSICNT").toString() : "");

dmperformance.setPernobusicnt(rs.getObject("PERNOBUSICNT") != null ? rs.getObject("PERNOBUSICNT").toString() : "");

dmperformance.setPerformancePercnt(rs.getObject("PERFORMANCE_PERCNT") != null ? rs.getObject("PERFORMANCE_PERCNT").toString() : "");

dmperformance.setAllmoney(rs.getObject("ALLMONEY") != null ? rs.getObject("ALLMONEY").toString() : "");

dmperformance.setPidfullpath(rs.getObject("PIDFULLPATH") != null ? rs.getObject("PIDFULLPATH").toString() : "");

dmperformance.setShopcnt(rs.getObject("SHOPCNT") != null ? rs.getObject("SHOPCNT").toString() : "");

dmperformance.setShopAllmoneyAvg(rs.getObject("SHOP_ALLMONEY_AVG") != null ? rs.getObject("SHOP_ALLMONEY_AVG").toString() : "");

dmperformance.setPerAllmoneyAvg(rs.getObject("PER_ALLMONEY_AVG") != null ? rs.getObject("PER_ALLMONEY_AVG").toString() : "");

dmperformance.setShopPerformanceAvg(rs.getObject("SHOP_PERFORMANCE_AVG") != null ? rs.getObject("SHOP_PERFORMANCE_AVG").toString() : "");

dmperformance.setCntPerformanceAvg(rs.getObject("CNT_PERFORMANCE_AVG") != null ? rs.getObject("CNT_PERFORMANCE_AVG").toString() : "");

dmperformance.setPerPerformanceAvg(rs.getObject("PER_PERFORMANCE_AVG") != null ? rs.getObject("PER_PERFORMANCE_AVG").toString() : "");

dmperformance.setPerformanceCommissionRate(rs.getObject("PERFORMANCE_COMMISSION_RATE") != null ? rs.getObject("PERFORMANCE_COMMISSION_RATE").toString() : "");

dmperformance.setDimorgName(rs.getObject("DIMORG_NAME") != null ? rs.getObject("DIMORG_NAME").toString() : "");

dmperformance.setPerformanceDictareaCnt(rs.getObject("PERFORMANCE_DICTAREA_CNT") != null ? rs.getObject("PERFORMANCE_DICTAREA_CNT").toString() : "");

dmperformance.setPerformanceDictareaRate(rs.getObject("PERFORMANCE_DICTAREA_RATE") != null ? rs.getObject("PERFORMANCE_DICTAREA_RATE").toString() : "");

dmperformance.setPerformanceAreaCnt(rs.getObject("PERFORMANCE_AREA_CNT") != null ? rs.getObject("PERFORMANCE_AREA_CNT").toString() : "");

dmperformance.setPerformanceAreaRate(rs.getObject("PERFORMANCE_AREA_RATE") != null ? rs.getObject("PERFORMANCE_AREA_RATE").toString() : "");

dmperformance.setDiscountCommissionRate(rs.getObject("DISCOUNT_COMMISSION_RATE") != null ? rs.getObject("DISCOUNT_COMMISSION_RATE").toString() : "");

dmperformance.setNoDealCnt(rs.getObject("NO_DEAL_CNT") != null ? rs.getObject("NO_DEAL_CNT").toString() : "");

dmperformance.setNoDealRate(rs.getObject("NO_DEAL_RATE") != null ? rs.getObject("NO_DEAL_RATE").toString() : "");

dmperformance.setDealCnt(rs.getObject("DEAL_CNT") != null ? rs.getObject("DEAL_CNT").toString() : "");

dmperformance.setDealRate(rs.getObject("DEAL_RATE") != null ? rs.getObject("DEAL_RATE").toString() : "");

dmperformance.setCntRentAvg(rs.getObject("CNT_RENT_AVG") != null ? rs.getObject("CNT_RENT_AVG").toString() : "");

dmperformance.setRentAvgRate(rs.getObject("RENT_AVG_RATE") != null ? rs.getObject("RENT_AVG_RATE").toString() : "");

dmperformance.setCommissionMoney(rs.getObject("COMMISSION_MONEY") != null ? rs.getObject("COMMISSION_MONEY").toString() : "");

dmperformance.setDiscountCommissionMoney(rs.getObject("DISCOUNT_COMMISSION_MONEY") != null ? rs.getObject("DISCOUNT_COMMISSION_MONEY").toString() : "");

dmperformance.setPerBusi(rs.getObject("PER_BUSI") != null ? rs.getObject("PER_BUSI").toString() : "");

dmperformance.setShopcnt1(rs.getObject("SHOPCNT1") != null ? rs.getObject("SHOPCNT1").toString() : "");

dmperformance.setAreacnt1(rs.getObject("AREACNT1") != null ? rs.getObject("AREACNT1").toString() : "");

dmperformance.setDictcnt1(rs.getObject("DICTCNT1") != null ? rs.getObject("DICTCNT1").toString() : "");

dmperformance.setShopcnt2(rs.getObject("SHOPCNT2") != null ? rs.getObject("SHOPCNT2").toString() : "");

dmperformance.setAreacnt2(rs.getObject("AREACNT2") != null ? rs.getObject("AREACNT2").toString() : "");

dmperformance.setDictcnt2(rs.getObject("DICTCNT2") != null ? rs.getObject("DICTCNT2").toString() : "");

dmperformance.setActperbusi2(rs.getObject("ACTPERBUSI2") != null ? rs.getObject("ACTPERBUSI2").toString() : "");

dmperformance.setCommissionCMoney(rs.getObject("COMMISSION_C_MONEY") != null ? rs.getObject("COMMISSION_C_MONEY").toString() : "");

dmperformance.setSellCAmount(rs.getObject("SELL_C_AMOUNT") != null ? rs.getObject("SELL_C_AMOUNT").toString() : "");

dmperformance.setRate27Cnt(rs.getObject("RATE_27_CNT") != null ? rs.getObject("RATE_27_CNT").toString() : "");

dmperformance.setRate25Cnt(rs.getObject("RATE_25_CNT") != null ? rs.getObject("RATE_25_CNT").toString() : "");

dmperformance.setRate20Cnt(rs.getObject("RATE_20_CNT") != null ? rs.getObject("RATE_20_CNT").toString() : "");

dmperformance.setRate15Cnt(rs.getObject("RATE_15_CNT") != null ? rs.getObject("RATE_15_CNT").toString() : "");

dmperformance.setRateOtherCnt(rs.getObject("RATE_OTHER_CNT") != null ? rs.getObject("RATE_OTHER_CNT").toString() : "");

dmperformance.setAllReachPerMoney(rs.getObject("ALL_REACH_PER_MONEY") != null ? rs.getObject("ALL_REACH_PER_MONEY").toString() : "");

dmperformance.setAllNoReachPerMoney(rs.getObject("ALL_NO_REACH_PER_MONEY") != null ? rs.getObject("ALL_NO_REACH_PER_MONEY").toString() : "");

dmperformance.setAllReachPerCnt(rs.getObject("ALL_REACH_PER_CNT") != null ? rs.getObject("ALL_REACH_PER_CNT").toString() : "");

dmperformance.setAllNoReachPerCnt(rs.getObject("ALL_NO_REACH_PER_CNT") != null ? rs.getObject("ALL_NO_REACH_PER_CNT").toString() : "");

dmperformance.setAllReachShopMoney(rs.getObject("ALL_REACH_SHOP_MONEY") != null ? rs.getObject("ALL_REACH_SHOP_MONEY").toString() : "");

dmperformance.setAllNoReachShopMoney(rs.getObject("ALL_NO_REACH_SHOP_MONEY") != null ? rs.getObject("ALL_NO_REACH_SHOP_MONEY").toString() : "");

dmperformance.setAllReachShopCnt(rs.getObject("ALL_REACH_SHOP_CNT") != null ? rs.getObject("ALL_REACH_SHOP_CNT").toString() : "");

dmperformance.setAllNoReachShopCnt(rs.getObject("ALL_NO_REACH_SHOP_CNT") != null ? rs.getObject("ALL_NO_REACH_SHOP_CNT").toString() : "");

dmperformance.setPerformanceReachPerMoney(rs.getObject("PERFORMANCE_REACH_PER_MONEY") != null ? rs.getObject("PERFORMANCE_REACH_PER_MONEY").toString() : "");

dmperformance.setPerformanceNoReachPerMoney(rs.getObject("PERFORMANCE_NO_REACH_PER_MONEY") != null ? rs.getObject("PERFORMANCE_NO_REACH_PER_MONEY").toString() : "");

dmperformance.setPerformanceReachPerCnt(rs.getObject("PERFORMANCE_REACH_PER_CNT") != null ? rs.getObject("PERFORMANCE_REACH_PER_CNT").toString() : "");

dmperformance.setPerformanceNoReachPerCnt(rs.getObject("PERFORMANCE_NO_REACH_PER_CNT") != null ? rs.getObject("PERFORMANCE_NO_REACH_PER_CNT").toString() : "");

dmperformance.setPerformanceReachShopMoney(rs.getObject("PERFORMANCE_REACH_SHOP_MONEY") != null ? rs.getObject("PERFORMANCE_REACH_SHOP_MONEY").toString() : "");

dmperformance.setPerformanceNoReaShopMoney(rs.getObject("PERFORMANCE_NO_REA_SHOP_MONEY") != null ? rs.getObject("PERFORMANCE_NO_REA_SHOP_MONEY").toString() : "");

dmperformance.setPerformanceReachShopCnt(rs.getObject("PERFORMANCE_REACH_SHOP_CNT") != null ? rs.getObject("PERFORMANCE_REACH_SHOP_CNT").toString() : "");

dmperformance.setPerformanceNoReachShopCnt(rs.getObject("PERFORMANCE_NO_REACH_SHOP_CNT") != null ? rs.getObject("PERFORMANCE_NO_REACH_SHOP_CNT").toString() : "");

dmperformance.setInsideDiscount(rs.getObject("INSIDE_DISCOUNT") != null ? rs.getObject("INSIDE_DISCOUNT").toString() : "");

dmperformance.setInsideDiscountRate(rs.getObject("INSIDE_DISCOUNT_RATE") != null ? rs.getObject("INSIDE_DISCOUNT_RATE").toString() : "");

dmperformance.setOutsideDiscount(rs.getObject("OUTSIDE_DISCOUNT") != null ? rs.getObject("OUTSIDE_DISCOUNT").toString() : "");

dmperformance.setOutsideDiscountRate(rs.getObject("OUTSIDE_DISCOUNT_RATE") != null ? rs.getObject("OUTSIDE_DISCOUNT_RATE").toString() : "");

dmperformance.setLitigationCompensation(rs.getObject("LITIGATION_COMPENSATION") != null ? rs.getObject("LITIGATION_COMPENSATION").toString() : "");

dmperformance.setLitigationCompensationRate(rs.getObject("LITIGATION_COMPENSATION_RATE") != null ? rs.getObject("LITIGATION_COMPENSATION_RATE").toString() : "");

dmperformance.setNoLitigationCompensation(rs.getObject("NO_LITIGATION_COMPENSATION") != null ? rs.getObject("NO_LITIGATION_COMPENSATION").toString() : "");

dmperformance.setNoLitigationCompensaRate(rs.getObject("NO_LITIGATION_COMPENSA_RATE") != null ? rs.getObject("NO_LITIGATION_COMPENSA_RATE").toString() : "");

dmperformance.setCancellationListMoney(rs.getObject("CANCELLATION_LIST_MONEY") != null ? rs.getObject("CANCELLATION_LIST_MONEY").toString() : "");

dmperformance.setCancellationListRate(rs.getObject("CANCELLATION_LIST_RATE") != null ? rs.getObject("CANCELLATION_LIST_RATE").toString() : "");

dmperformance.setNormalListMoney(rs.getObject("NORMAL_LIST_MONEY") != null ? rs.getObject("NORMAL_LIST_MONEY").toString() : "");

dmperformance.setNormalListRate(rs.getObject("NORMAL_LIST_RATE") != null ? rs.getObject("NORMAL_LIST_RATE").toString() : "");

dmperformance.setCancellationMoney(rs.getObject("CANCELLATION_MONEY") != null ? rs.getObject("CANCELLATION_MONEY").toString() : "");

dmperformance.setCancellationRate(rs.getObject("CANCELLATION_RATE") != null ? rs.getObject("CANCELLATION_RATE").toString() : "");

dmperformance.setTravelMoney(rs.getObject("TRAVEL_MONEY") != null ? rs.getObject("TRAVEL_MONEY").toString() : "");

dmperformance.setTravelRate(rs.getObject("TRAVEL_RATE") != null ? rs.getObject("TRAVEL_RATE").toString() : "");

dmperformance.setMaidMoney(rs.getObject("MAID_MONEY") != null ? rs.getObject("MAID_MONEY").toString() : "");

dmperformance.setMaidRate(rs.getObject("MAID_RATE") != null ? rs.getObject("MAID_RATE").toString() : "");

dmperformance.setPerformanceRate(rs.getObject("PERFORMANCE_RATE") != null ? rs.getObject("PERFORMANCE_RATE").toString() : "");

dmperformance.setRentRate(rs.getObject("RENT_RATE") != null ? rs.getObject("RENT_RATE").toString() : "");

dmperformance.setGuarRate(rs.getObject("GUAR_RATE") != null ? rs.getObject("GUAR_RATE").toString() : "");

dmperformance.setNewhouseRate(rs.getObject("NEWHOUSE_RATE") != null ? rs.getObject("NEWHOUSE_RATE").toString() : "");

dmperformance.setPerformanceDictMoneyRate(rs.getObject("PERFORMANCE_DICT_MONEY_RATE") != null ? rs.getObject("PERFORMANCE_DICT_MONEY_RATE").toString() : "");

dmperformance.setPerformanceAreaMoneyRate(rs.getObject("PERFORMANCE_AREA_MONEY_RATE") != null ? rs.getObject("PERFORMANCE_AREA_MONEY_RATE").toString() : "");

dmperformance.setPerformanceSAreaMoneyRate(rs.getObject("PERFORMANCE_S_AREA_MONEY_RATE") != null ? rs.getObject("PERFORMANCE_S_AREA_MONEY_RATE").toString() : "");

dmperformance.setRentHRate(rs.getObject("RENT_H_RATE") != null ? rs.getObject("RENT_H_RATE").toString() : "");

dmperformance.setRentCRate(rs.getObject("RENT_C_RATE") != null ? rs.getObject("RENT_C_RATE").toString() : "");

dmperformance.setNidfullpath(rs.getObject("NIDFULLPATH") != null ? rs.getObject("NIDFULLPATH").toString() : "");

dmperformance.setNpidfullpath(rs.getObject("NPIDFULLPATH") != null ? rs.getObject("NPIDFULLPATH").toString() : "");

dmperformance.setProcessed(rs.getObject("PROCESSED") != null ? rs.getObject("PROCESSED").toString() : "");

dmperformance.setStatus("1");returndmperformance;

}

});for(DmPerformance dmperformance : dmperformances) {try{

objs.add(StatUtils.objectToMap(dmperformance));

}catch(Exception e) {

e.printStackTrace();

}

}

}try{

String clusterNames=AppPropUtil.getPropValue(AppPropUtil.Keys.BI_CLUSTER_NAME);if (!CollectionUtils.isEmpty(objs)) {//保证结果集不为空

boolean batchInsert =searchService.batchInsert(clusterNames,indexName1,StatContant.CATEGORY_TYPE_MAP.get(category), objs);if(batchInsert) {//如果成功返回200更新中间表状态为已处理

logger.error("------------------开始更新"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据状态------------------------");

StringBuffer sql= new StringBuffer("update "+StatContant.CATEGORY_TYPE_MAP.get(category)+"_batch t set t.status = 1 where ");

ArrayList midLists = new ArrayList();if(category.equals(StatContant.CATEGORY_TYPE_HOUSE)) {//房源

for(DmHouseBase dmHouseBase : dmHouseBases) {

midLists.add(dmHouseBase.getMid());

}

}else if(category.equals(StatContant.CATEGORY_TYPE_CUSTORMER)) {//客源

for(DmCustomer customer : dmCustomers) {

midLists.add(customer.getMid());

}

}else if(category.equals(StatContant.CATEGORY_TYPE_PERFORMANCE)) {//业绩

for(DmPerformance dmperformance : dmperformances) {

midLists.add(dmperformance.getMid());

}

}

String sqlbuffer= StatUtils.chgListStrToInStr("t.mid", midLists);

sql.append(sqlbuffer);int[] batchUpdate = biJdbcTemplate.batchUpdate(newString[]{sql.toString()});

logger.error("------------------结束更新"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据状态共：" + batchUpdate[0] + "条-----------------------");

}else{//es插入失败，强制回滚事物，干掉这一批插入的全部数据

logger.error("------------------batch插入es报错，强制回滚事物，批量删除开始------------------------");

StringBuffer buffer= newStringBuffer();if(category.equals(StatContant.CATEGORY_TYPE_HOUSE)) {//房源

for(DmHouseBase dmHouseBase : dmHouseBases) {

buffer.append(dmHouseBase.getMid()).append(" ");

}

}else if(category.equals(StatContant.CATEGORY_TYPE_CUSTORMER)) {//客源

for(DmCustomer customer : dmCustomers) {

buffer.append(customer.getMid()).append(" ");

}

}else if(category.equals(StatContant.CATEGORY_TYPE_PERFORMANCE)) {//业绩

for(DmPerformance dmperformance : dmperformances) {

buffer.append(dmperformance.getMid()).append(" ");

}

}

String midsString= buffer.substring(0, buffer.length() - 1);

Map conditions = new HashMap();

conditions.put("mid", midsString);boolean deleteRecordsByConditions =searchService.deleteRecordsByConditions(clusterNames, indexName1,StatContant.CATEGORY_TYPE_MAP.get(category), conditions);if(deleteRecordsByConditions) {

logger.error("------------------batch插入es报错，强制回滚事物，批量删除结束------------------------");

}

}

}

}catch(Exception e) {

e.printStackTrace();

}

}

});

}

es.shutdown();try{boolean loop = true;do { //等待所有任务完成

loop = !es.awaitTermination(2, TimeUnit.SECONDS); //阻塞，直到线程池里所有任务结束

} while(loop);

}catch(Exception e) {

e.printStackTrace();

}

}

logger.error("------------------同步BI两个月"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据结束：结束时间" + DateUtil.getDefaultDate() + "-----------------------");long end =System.currentTimeMillis();

logger.error("同步BI两个月"+StatContant.CATEGORY_TYPE_MAP.get(category)+"中间表数据共耗时" + (end - begin) + "ms");//写入跑批日志记录表

String deletesql = "delete from sys_stat_jobs_exe_detail sd where sd.dm_time = ? and sd.dm_type = ?";

Object args[]= new Object[]{1, StatContant.CATEGORY_TYPE_MAP.get(category)};

jdbcTemplate.update(deletesql, args);

String insertsql= "insert into sys_stat_jobs_exe_detail values(?,?,?,?,?)";int exectime = (int) (end -begin);

Object args1[]= new Object[]{StatContant.CATEGORY_TYPE_MAP.get(category), "1", new Date(begin), newDate(end), exectime};int count =jdbcTemplate.update(insertsql, args1);if (count > 0) {

logger.error("插入跑批日志成功,插入类型" + StatContant.CATEGORY_TYPE_MAP.get(category) + "---" + "1");

}

}

}

}

weixin_39866087
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
分页多线程处理大批量数据
在青海看海的企鹅的博客
03-20 525
因为需要从一个返利明细表中获取大量的数据，生成返利报告，耗时相对较久，作为后台任务执行。但是后台任务如果不用多线程处理，也会要很长时间才能处理完。另外考虑到数据量大，不能一次查询所有数据在内存中处理，为了防止内存溢出，分页查询数据，然后分批次多线程处理。
java多线程实现大批量数据导入源码
09-29
java多线程实现大批量数据切分成指定份数的数据，然后多线程处理入库或者导出，线程的个数和每份数据的数量都可以控制
Java多线程获取网页数据并更新到数据库
最新发布
weixin_42260782的博客
06-07 774
创建线程的工厂，通过自定义的线程工厂可以给每个新建的线程设置一个具有识别度的线程名Executors静态工厂里默认的threadFactory，线程的命名规则是“pool-数字-thread-数字”，这里使用默认。如果当前阻塞队列满了，且继续提交任务，则创建新的线程执行任务，前提是当前线程数小于maximumPoolSize，这里使用的。这里需要注意的是，一般网上默认的例子都是使用第一个，这里我在使用的时候，由于参数设置的不合理，导致直接抛出异常，终止了程序，这里我是用的是第二种。
Java多线程大批量同步数据（分页）
宋发元
09-27 6462
背景 最近遇到个功能，两个月有300w+的数据，之后还在累加，因一开始该数据就全部存储在mysql表，现需要展示在页面，还需要关联另一张表的数据，而且产品要求页面的查询条件多达20个条件，最终，这个功能卡的要死，基本查不出来数据。 最后是打算把这两张表的数据同时存储到MongoDB中去，以提高查询效率。 一开始同步的时候，采用单线程，循环以分页的模式去同步这两张表数据，结果是…一晚上，只同步了30...
java使用多线程及分页查询数据量很大的数据
胖蛤蟆的博客
03-10 1910
java使用多线程及分页查询数据量很大的数据 import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import.
多线程分页处理数据（主线程阻塞直到所有子线程处理完毕并接收返回结果）
被活捉的花生米的博客
09-11 1454
方案一 备注：将数据分页，以FixedThreadPool（定长线程池）+CountDownLatch（并发流程控制）+Callable(有返回值的多线程) + Future（接收Callable的返回值）的形式进行多线程处理数据。其中CountDownLatch的工作原理：在初始化时传入子线程数量，主线程调用await()方法阻塞主线程直到CountDownLatch的技术为0，而使减计数的方法...
数据同步--分页中开启多线程进行数据同步（1）
涵山韵海的博客
04-25 950
在进行接口之间数据同步时候，数据量大的话，定时任务或者接口会执行很久，需要优化的话，思路可从以下几点考虑： 1.先分页处理。 2.分页中进行多线程处理。 3.根据服务器配置，决定开启的多线程条数，一般的服务器配置建议开启10-15个。 代码逻辑参考如下： //数据总条数 int total = xxxxxx.selectCount(); //共查询多少次 int count = 0; if (total > pageSize
Java多线程结构_Java多线程结构_
09-30
Java中的多线程结构请大家观看不足的地方请指教，大家可以一语既出的观看和指导。
详解Java多线程处理List数据
08-26
主要介绍了Java多线程处理List数据，文中通过示例代码介绍的非常详细，对大家的学习或者工作具有一定的参考学习价值，需要的朋友们下面随着小编来一起学习学习吧
java多线程编程_java多线程_
09-29
1.讲解了Java多线程的基础， 包括Thread类的核心API的使用。2.讲解了在多线程中对并发访问的控制， 主要就是synchronized的使用， 由于此关键字在使用上非常灵活， 所以书中用了很多案例来介绍此关键字的使用， 为...
duoxianchengxiazai.rar_Java 多线程 下载_java 多线程下载
09-23
多线程下载，随便做的，有一定的局限性，仅供参考
线程池-分页处理数据，取出重复数据的问题解决
Shengzhiying的博客
10-09 1008
线程池分页处理-重复处理数据问题解决
java多线程查询_java使用多线程及分页查询数据量很大的数据
weixin_36373949的博客
02-12 1291
调用方法： import org.springframework.beans.factory.annotation.Autowired; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.Callable; import java.util.c...
java 多线程处理分页数据_java多线程数据分页处理实例讲解
weixin_35180056的博客
02-13 1603
免费资源网,https://freexyz.cn/在数据的最终结果上，我们能够通过分类的方法，准备的筛选出不同类别结果的信息。这里我们发散一下思维，在Java中对于数据大量处理的，多线程是一个非常常见的代表，我们可以用分页来处理多线程的数据问题。下面我们对分类的类型进行了解，然后带来两种分页在多线程的逻辑。1.常见的分页类型传统的：采用传统的分页方式，可以明确的获取数据信息，如有多少条数据，分多少...
【无标题】java多线程分批同步数据设计与实现（转载）
zhudauy的博客
11-12 695
 已识乾坤大，犹怜草木青。          ——旷怡亭口占 文章目录 前言 设计思路 代码实践 前言 最近接到一个任务，将mysql中的数据同步到elasticsearch中，要求异步执行，接口不必返回结果，接到请求后后台默默执行就行了。这种情况下其实单线程一页一页去读取写入就可以了，因为不必立刻返回请求结果后台执行5分钟还是十分钟只要能把数据加入到es中就可以了。 但是，身为一个优秀的程序员 怎么只能考虑功能的实现不考虑效率问题呢😎😎😎（时间允许的情况下考虑效率对
数据分片，多线程处理
qq_25817753的博客
04-03 1391
场景：mysql有100w条数据，要根据id更新到Elasticsearch。 处理方式：100w 条数据分页查询，每次查询1w条数据，再将这1w条数据分成10个分片，每个分片1000条数据，每个分片的1000数据由一条线程去执行处理，相当于10个分片由10个线程处理。 线程池工具类 /** * 线程池工具类 */ public class ThreadUtil { /** * 线程池实例 */ private static ThreadPoolExec.
批量数据分解多线程并行处理方案
沧海桑田的博客
03-09 802
项目中存在很多批量数据入库业务，通常这些业务，首先需要对每一条数据进行格式校验，如数据类型、必填性、长度、枚举、范围、关联性等，其次需要对每一条数据进行业务逻辑校验，如重复性、真实性、数据权限、锁权限、写入权限等。这些业务，涉及到文件IO、网络IO、本地计算，各子过程的数据量较大时通常需要很长的耗时，尤其是传统单线程处理时。 多线程或多进程处理批量数据可以提高效率，多进程处理相对比较复杂，本方案采用更常用的解决方案，即多线程处理。...
java多线程处理大批量数据
06-06
利用Java多线程的特性，我们可以很方便地处理大批量数据。 首先，我们可以使用Java中的Executor框架来管理多个任务的执行。Executor框架可以管理线程的生命周期，并提供了一些便捷的创建线程池的方法，可以有效地...

“相关推荐”对你有帮助么？

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值