一、
1、批数据处理
1.1 分批查询
1 mongdb查询count 数量极快!
2 limit分批查询禁止使用分页,应使用比较id查询
/*
* 查询mongodb中已经存在的指定SourceType的药品列表list
*
*/
public Map<Long, String> getCurrentErpMedicineMap() throws Exception {
DocDB mongo = DocDB.mongo(PlatformType.HIS.domain());
Document query = new Document();
query.append("SourceType", PlatformType.ERP.getValue());
// count
int count = mongo.mongoCount(SyncShopMedicineEntity.class, query);
if (count == 0)
return null;
int size = 16;
int sizeTemp = (int) (count/ 0.75 + 1);
size = size > sizeTemp ? size : sizeTemp;
Map<Long, String> result = new HashMap<>(size);
int limit = 20000;
List<Map<String, Object>> list = null;
Long curMaxId = 0L;
int curCount = 0;
Map<String, String> includeFieldAlias = new HashMap<>();
includeFieldAlias.put("SourceId", "SourceId");
includeFieldAlias.put("CatalogCode", "CatalogCode");
while(curCount < count){
// 比较id查询
list = mongo.mongoQueryRows(SyncShopMedicineEntity.class, query.append("_id", new Document("$gt",curMaxId)),includeFieldAlias, null, 0, limit);
if(list!=null&&!list.isEmpty()){
curMaxId = ObjectTransfer.longValue(list.get(list.size()-1).get("_id"));
for (Map<String, Object> map : list) {
Long key = ObjectTransfer.longValue(map.get("SourceId"));
String value = ObjectTransfer.stringValue(map.get("CatalogCode"));
if(key==null||value==null||key.longValue()==0||value.isEmpty())
continue;
result.put(key, value);
}
}
curCount+=limit;
}
return result;
}