记一次数据处理效率提升

业务处理过程：
1、读取轨迹数据(4658520条记录)
2、请求第三方轨迹匹配服务，进行道路绑定
3、保存绑路结果入库

实现：
/*
**4658520条记录
**第一种方法：按订单获取轨迹，调用轨迹匹配服务，根据返回数据单点更新
**90条/分
**MacBook Pro (13-inch, 2020, Four Thunderbolt 3 ports) 2 GHz 四核Intel Core i5 16 GB 3733 MHz LPDDR4X
*/
List<String> orderIdList = trackCoorInfoBakMapper.getOrderId();
List<String> trackList = new ArrayList<>();
//根据oderId拼接请求对象
for(String orderId:orderIdList){
List<TrackCoorInfoBak> trackCoorInfoList = trackCoorInfoBakService.lambdaQuery().eq(TrackCoorInfoBak::getOrderId,orderId).gt(TrackCoorInfoBak::getOrderId,"966164690109464576").orderByAsc(TrackCoorInfoBak::getLocTime).list();
for(TrackCoorInfoBak trackCoorInfo:trackCoorInfoList){
//{"coorType":"wgs84", "originalTrack":["100.5996747,13.8047994,1603644352,1.2980503,305.71634",
//"100.5991847,13.8047918,1603644355,0.0,305.71634",
//"100.5990894,13.8052371,1603644361,5.4530067,307.7255",
//"100.5991361,13.8051712,1603644367,6.5436397,311.42847",
//"100.5991901,13.8052015,1603644370,6.042252,309.5075"]
//}
trackList.add(trackCoorInfo.getLongitude()+","+trackCoorInfo.getLatitude()+","+(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).parse(trackCoorInfo.getLocTime()).getTime())+","+"0"+","+trackCoorInfo.getBearing());
}
System.out.println(trackList);
TrackMatchReq trackMatchReq = new TrackMatchReq("wgs84",trackList);
//请求匹配结果
String matchResult = roadMatchService.getMatchResult(JSON.toJSONString(trackMatchReq));
//解析返回data
JSONObject object = JSONObject.parseObject(matchResult);
JSONObject dataObject = object.getJSONObject("data");
JSONArray array = dataObject.getJSONArray("data");
//根据返回结果更新数据
int i=0;
for(TrackCoorInfoBak trackCoorInfo:trackCoorInfoList){
UpdateWrapper<TrackCoorInfoBak> updateWrapper = new UpdateWrapper<>();
updateWrapper.eq("id",Integer.parseInt(trackCoorInfo.getId())).set("tt_link", array.getJSONObject(i).get("link_id"));
System.out.println(trackCoorInfo.getLatitude());
System.out.println(trackCoorInfo.getLongitude());
System.out.println(trackCoorInfo.getLocTime());
System.out.println(trackCoorInfo.getOrderId());
System.out.println(trackCoorInfo.getId());
System.out.println(array.getJSONObject(i).get("link_id"));
Integer rows = trackCoorInfoBakMapper.update(null, updateWrapper);
//trackCoorInfoService.lambdaUpdate().eq(TrackCoorInfo::getId,trackCoorInfo.getId()).set(TrackCoorInfo::getTtLink,array.getJSONObject(i).get("link_id")).update();
i++;
}
}


/*
**第二种方法：按订单获取轨迹，调用轨迹匹配服务；按订单轨迹批量更新，id建索引
**9057条/分
**MacBook Pro (13-inch, 2020, Four Thunderbolt 3 ports) 2 GHz 四核Intel Core i5 16 GB 3733 MHz LPDDR4X
*/

//获取所有orderId
List<String> orderIdList = trackCoorInfoBakMapper.getOrderId();
//根据oderId拼接请求对象
for(String orderId:orderIdList){
//List<TrackCoorInfoBak> trackCoorInfoList = trackCoorInfoBakService.lambdaQuery().eq(TrackCoorInfoBak::getOrderId,orderId).gt(TrackCoorInfoBak::getOrderId,"966164690109464576").orderByAsc(TrackCoorInfoBak::getLocTime).list();
List<TrackCoorInfoBak> trackCoorInfoList = trackCoorInfoBakService.lambdaQuery().eq(TrackCoorInfoBak::getOrderId,orderId).orderByAsc(TrackCoorInfoBak::getLocTime).list();
List<String> trackList = new ArrayList<>();
for(TrackCoorInfoBak trackCoorInfo:trackCoorInfoList){
//{"coorType":"wgs84", "originalTrack":["100.5996747,13.8047994,1603644352,1.2980503,305.71634",
//"100.5991847,13.8047918,1603644355,0.0,305.71634",
//"100.5990894,13.8052371,1603644361,5.4530067,307.7255",
//"100.5991361,13.8051712,1603644367,6.5436397,311.42847",
//"100.5991901,13.8052015,1603644370,6.042252,309.5075"]
//}
trackList.add(trackCoorInfo.getLongitude()+","+trackCoorInfo.getLatitude()+","+(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).parse(trackCoorInfo.getLocTime()).getTime())+","+"0"+","+trackCoorInfo.getBearing());
}
System.out.println(trackList);
TrackMatchReq trackMatchReq = new TrackMatchReq("wgs84",trackList);
//请求匹配结果
String matchResult = roadMatchService.getMatchResult(JSON.toJSONString(trackMatchReq));
//解析返回data
JSONObject object = JSONObject.parseObject(matchResult);
JSONObject dataObject = object.getJSONObject("data");
JSONArray array = dataObject.getJSONArray("data");
//根据返回结果更新数据
int i=0;
List<TrackCoorInfoBak> trackCoorInfoBakUpdateList = new ArrayList<>();
for(TrackCoorInfoBak trackCoorInfo:trackCoorInfoList){
if(array.getJSONObject(i).get("link_id") == null){
trackCoorInfo.setTtLink("-1");
}else {
trackCoorInfo.setTtLink(array.getJSONObject(i).get("link_id").toString());
}
System.out.println(trackCoorInfo.getId()+","+trackCoorInfo.getOrderId()+","+trackCoorInfo.getLongitude()+","
+trackCoorInfo.getLatitude()+","+trackCoorInfo.getLocTime()+",0.0,"+trackCoorInfo.getBearing());
System.out.println(array.getJSONObject(i).get("link_id"));
trackCoorInfoBakUpdateList.add(trackCoorInfo);
i++;
}
trackCoorInfoBakService.updateBatchById(trackCoorInfoBakUpdateList);
System.out.println("finish processing trackorder:"+ orderId);
}

/*
**第三种方法：按订单获取轨迹，调用轨迹匹配服务，按轨迹批量更新，id建索引，异步调用 springboot @asnc
**95072条/分
**MacBook Pro (13-inch, 2020, Four Thunderbolt 3 ports) 2 GHz 四核Intel Core i5 16 GB 3733 MHz LPDDR4X
*/

在application中增加：
@EnableAsync

controller：
@GetMapping(value = "/trackMatchBatch")
@ApiOperation(value = "trackMatchBatch")
public ResponseResult<String> trackMatchBatch() throws ParseException, InterruptedException {
//获取所有orderId
List<String> orderIdList = trackCoorInfoMapper.getOrderId();
//根据oderId拼接请求对象
for(String orderId : orderIdList){
mnrNetwGeoLinkService.getTrackMatch(orderId);
}

return ResponseResult.OK(true);
}

service：
@Async()
public void getTrackMatch(String orderId) throws ParseException {
//List<TrackCoorInfoBak> trackCoorInfoList = trackCoorInfoBakService.lambdaQuery().eq(TrackCoorInfoBak::getOrderId,orderId).gt(TrackCoorInfoBak::getOrderId,"966164690109464576").orderByAsc(TrackCoorInfoBak::getLocTime).list();
List<TrackCoorInfo> trackCoorInfoList = trackCoorInfoService.lambdaQuery().eq(TrackCoorInfo::getOrderId, orderId).orderByAsc(TrackCoorInfo::getLocTime).list();
List<String> trackList = new ArrayList<>();
for (TrackCoorInfo trackCoorInfo : trackCoorInfoList) {
//{"coorType":"wgs84", "originalTrack":["100.5996747,13.8047994,1603644352,1.2980503,305.71634",
//"100.5991847,13.8047918,1603644355,0.0,305.71634",
//"100.5990894,13.8052371,1603644361,5.4530067,307.7255",
//"100.5991361,13.8051712,1603644367,6.5436397,311.42847",
//"100.5991901,13.8052015,1603644370,6.042252,309.5075"]
//}
trackList.add(trackCoorInfo.getLongitude() + "," + trackCoorInfo.getLatitude() + "," + (new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH).parse(trackCoorInfo.getLocTime()).getTime()) + "," + "0" + "," + trackCoorInfo.getBearing());
}
System.out.println(trackList);
TrackMatchReq trackMatchReq = new TrackMatchReq("wgs84", trackList);
//请求匹配结果
String matchResult = roadMatchService.getMatchResult(JSON.toJSONString(trackMatchReq));
//解析返回data
JSONObject object = JSONObject.parseObject(matchResult);
if (object.get("ret").toString().compareTo("-1") == 0) {
System.out.println("trace match no result ret = -1;orderId is:" + orderId);
return;
}
JSONObject dataObject = object.getJSONObject("data");
JSONArray array = dataObject.getJSONArray("data");
if (array.size() == 0) {
System.out.println("trace match no result;orderId is:" + orderId);
return;
}
//根据返回结果更新数据
int i = 0;
List<TrackCoorInfo> trackCoorInfoUpdateList = new ArrayList<>();
for (TrackCoorInfo trackCoorInfo : trackCoorInfoList) {
System.out.println(i);
if (array.getJSONObject(i).get("link_id") == null) {
trackCoorInfo.setTtLink("-1");
} else {
trackCoorInfo.setTtLink(array.getJSONObject(i).get("link_id").toString());
}
System.out.println(trackCoorInfo.getId() + "," + trackCoorInfo.getOrderId() + "," + trackCoorInfo.getLongitude() + ","
+ trackCoorInfo.getLatitude() + "," + trackCoorInfo.getLocTime() + ",0.0," + trackCoorInfo.getBearing());
System.out.println(array.getJSONObject(i).get("link_id"));
trackCoorInfoUpdateList.add(trackCoorInfo);
i++;
}
trackCoorInfoService.updateBatchById(trackCoorInfoUpdateList);
System.out.println("finish processing trackorder:" + orderId);
}