前言
一切操作都应该以官方文档为准,mongodb官网文档地址: https://www.mongodb.com/docs/ ,网上关于java操作mongodb的文章偏少,而且有些乱。这篇文章是在项目中使用mongodb后的一些总结,希望能帮到大家。
1.创建mongodb客户端(这里注册了自定义解码器,方便直接存java对象)
String linkUrl = "mongodb://" + url;
if (hasAuth) { //需要用户名密码用下面的方式拼接url
linkUrl = "mongodb://" + username + ":" + password + "@" + url + "/" + databaseName;
}
CodecRegistry pojoCodecRegistry = CodecRegistries.fromRegistries(
MongoClientSettings.getDefaultCodecRegistry(),
CodecRegistries.fromProviders(PojoCodecProvider.builder().automatic(true).build()));
MongoClientSettings build = MongoClientSettings.builder()
.codecRegistry(pojoCodecRegistry)
.applyConnectionString(new ConnectionString(linkUrl))
.build();
MongoClient client = MongoClients.create(build);
2.批量写入对象数据
public static void write(MongoClient client, String databaseName, String collectionName, List<?> dataList) {
if (dataList != null && dataList.size() != 0) {
MongoDatabase database = client.getDatabase(databaseName);
MongoCollection<Document> collection = database.getCollection(collectionName);
List<WriteModel<Document>> writes = new ArrayList();
Iterator var7 = dataList.iterator();
while(var7.hasNext()) {
Object data = var7.next();
Map<String, Object> dataMap = ObjectToMapUtil.objectToMap(data);
writes.add(new InsertOneModel(new Document(dataMap)));
}
collection.bulkWrite(writes);
}
}
public static Map<String, Object> objectToMap(Object obj) {
Map<String, Object> map = new HashMap();
try {
Class<?> clazz = obj.getClass();
for (Field field : clazz.getDeclaredFields()) {
field.setAccessible(true);
map.put(field.getName(), field.get(obj));
}
}catch (Exception e){
log.info("ObjectToMapUtil.objectToMap 实体类转换成Map失败={}", JSON.toJSONString(e.getMessage()));
e.printStackTrace();
}
return map;
}
3.批量更新数据
private void batchUpdateData(MongoCollection<Document> collection, List<POJO> list) {
if (CollectionUtil.isEmpty(list)) {
return;
}
List<String> ids = list.stream().map(POJO::get_id).collect(Collectors.toList());
List<WriteModel<Document>> writes = new ArrayList();
for (int i = 0; i < list.size(); i++) {
Object data = list.get(i);
String id = ids.get(i);
Map<String, Object> dataMap = ObjectToMapUtil.objectToMap(data);
UpdateOneModel updateOneModel = new UpdateOneModel(Filters.eq("_id", id), new Document("$set", new Document(dataMap)), (new UpdateOptions()).upsert(true));
writes.add(updateOneModel);
}
collection.bulkWrite(writes);
}
4.查询
4.1.自定义条件封装,二维数组
private Document buildReportQuery(BasicPageBO pageBO) {
Document query = new Document();
List<List<Condition>> conditions = pageBO.getConditions();
if (CollectionUtil.isEmpty(conditions)) {
return query;
}
query = MongoOperateUtil.buildConditions(conditions);
return query;
}
//BasicPageBO是一个灵活的参数对象,里面有分页信息,非常好用
//条件参数对象
public static class Condition implements Serializable {
@ApiModelProperty("字段名")
private String field;
@ApiModelProperty("操作符,gt,gte,eq等")
private String operator;
@ApiModelProperty("值")
private String value;
@ApiModelProperty("逻辑条件,and、or等")
private String logic;
}
public static Document strEq(String field, String value) {
return new Document(field, value);
}
public static Document buildBson(BasicPageBO.Condition condition) {
String operator = condition.getOperator();
switch (operator) {
case "eq" :
return MongoOperateUtil.strEq(condition.getField(), condition.getValue());
case "ne" :
return MongoOperateUtil.strNe(condition.getField(), condition.getValue());
case "like" :
return MongoOperateUtil.like(condition.getField(), condition.getValue());
case "startWith" :
return MongoOperateUtil.leftLike(condition.getField(), condition.getValue());
case "endWith" :
return MongoOperateUtil.rightLike(condition.getField(), condition.getValue());
case "in" :
return MongoOperateUtil.strIn(condition.getField(), condition.getValue());
case "nin" :
return MongoOperateUtil.strNin(condition.getField(), condition.getValue());
case "isNull" :
return MongoOperateUtil.isNull(condition.getField(), null);
default: {
return null;
}
}
}
private static Document strNe(String field, String value) {
return new Document(field, new Document("$ne", value));
}
private static Document isNull(String field, String value) {
return new Document(field, value);
}
public static Document buildConditions(List<List<BasicPageBO.Condition>> conditionAll){
Document totalBson = null;
for (int i = 0; i < conditionAll.size(); i++) {
List<BasicPageBO.Condition> conditions = conditionAll.get(i);
Document oneBson = null;
String twoLogic = conditions.get(0).getLogic();
for (int j = 0; j < conditions.size(); j++) {
BasicPageBO.Condition condition = conditions.get(j);
String logic = condition.getLogic();
//加上前缀
condition.setField("dataMap." + condition.getField());
//单个构建
Document bson = MongoOperateUtil.buildBson(condition);
if (0 == j) {
oneBson = bson;
} else {
oneBson = MongoOperateUtil.conditionOn(oneBson, bson, logic);
}
}
if (0 == i) {
totalBson = oneBson;
} else {
totalBson = MongoOperateUtil.conditionOn(totalBson, oneBson, twoLogic);
}
}
return totalBson;
}
public static Document conditionOn(Document first, Document two, String logic) {
BasicDBList list = new BasicDBList();
list.add(first);
list.add(two);
if ("and".equals(logic)) {
return new Document("$and", list);
} else if("or".equals(logic)) {
return new Document("$or", list);
}
return null;
}
public static Document strIn(String field, String value) {
String[] valueArr = value.split(",");
List<String> vList = Arrays.asList(valueArr);
String operate = "$in";
BasicDBList vBasics = new BasicDBList();
vBasics.addAll(vList);
return new Document(field, new Document(operate, vBasics));
}
public static Document strNin(String field, String value) {
String[] valueArr = value.split(",");
List<String> vList = Arrays.asList(valueArr);
String operate = "$nin";
BasicDBList vBasics = new BasicDBList();
vBasics.addAll(vList);
return new Document(field, new Document(operate, vBasics));
}
public static Document like(String field, String value) {
Document dbo = new Document();
Pattern pattern = Pattern.compile("^.*" + value+ ".*$", Pattern.CASE_INSENSITIVE);
dbo.put(field, pattern);
return dbo;
}
public static Document leftLike(String field, String value) {
Document dbo = new Document();
Pattern pattern = Pattern.compile("^" + Pattern.quote(value) + ".*", Pattern.CASE_INSENSITIVE);
dbo.put(field, pattern);
return dbo;
}
public static Document rightLike(String field, String value) {
Document dbo = new Document();
Pattern pattern = Pattern.compile(".*" + Pattern.quote(value) + "$", Pattern.CASE_INSENSITIVE);
dbo.put(field, pattern);
return dbo;
}
4.2 单表查询
Document condition = new Document();
condition.put(k,v)//加各种条件
计数 db.getCollection(表名).countDocuments(condition)
分页查询 db.getCollection(表名).find(condition).skip(skip).limit(pageSize);
4.3 多表lookup关联查询(大数据量时,已经放弃,分页查询求count时性能极慢(30w数据,耗时30秒左右))
4.3.1 执行的各个阶段
l
o
o
k
u
p
−
>
lookup->
lookup−>match->
g
r
o
u
p
−
>
group->
group−>sort->
u
n
w
i
n
d
−
>
unwind->
unwind−>project->
s
k
i
p
−
>
skip->
skip−>limit
leftjoin用到$unwind preserveNullAndEmptyArrays: true
4.3.2 如下是一个案例:
int current = (int) pageBO.getCurrent();
int pageSize = (int) pageBO.getSize();
int skip = pageSize * (current - 1);
//以key(某个字段)关联
String joinField = "key";
//注意,这个字段名需要与返回数据对象中的(关联对象)属性名称对应,不然json解析不出数据
String tableAlias = "linkDto";
// 联合查询条件
List<Bson> aggregationList = new LinkedList<Bson>();
//主表查询条件
Bson matchFilter = Aggregates.match(match);
aggregationList.add(matchFilter);
//从表名、主表连接字段、从表连接字段、别名
aggregationList.add(
Aggregates.lookup(param.getTableId(), joinField, joinField, tableAlias)
);
BasicDBObject bson = new BasicDBObject();
bson.put(statusKey, param.getStatus);
aggregationList.add(
Aggregates.unwind("$linkDto", new UnwindOptions().preserveNullAndEmptyArrays(true))
);
aggregationList.add(
Aggregates.match(bson)
);
List<Bson> countBson = new ArrayList<>(aggregationList);
countBson.add(
Aggregates.group("_id", new BsonField("count", new BasicDBObject("$sum", 1)))
);
//分组聚会统计数量
AggregateIterable<Document> countAgg = util.getDb().getCollection(表名).aggregate(countBson).batchSize(1);
long total = 0;
Document first = countAgg.first();
if (Objects.nonNull(first) && Objects.nonNull(first.get("count"))) {
total = (long) (Integer) first.get("count");
}
aggregationList.add(Aggregates.skip(skip));
aggregationList.add(Aggregates.limit(pageSize));
AggregateIterable<Document> labelList = util.getDb().getCollection(表名).aggregate(aggregationList);
//提取和转换结果集
for (Document dr : labelList) {
dr.toJson();
//todo .....
}
PageDto<POJOView> pageBean = new PageDto<>();
pageBean.setTotal(total);
pageBean.setRecords(results);
pageBean.setCurrent(current);
pageBean.setSize(pageSize);
## 5.命令行相关查询(直接放到navicat上执行)
例子1,总数
```javascript
db.getCollection("主表名").aggregate([
{
$lookup: {
from: "从表名", // 关联的集合名称
localField: "key", // 本地字段,主表关联字段,建索引也没有卵用 集合中用于关联的字段
foreignField: "key", // 外部字段,从表关联字段 集合中用于关联的字段, 意思是主表key=从表key,若不是相等,可以用管线的形式(构建其他条件,最后别用,真是难用)
as: "report" // 关联结果存放的字段名
}
},
{
$unwind: {
path: "$report",
preserveNullAndEmptyArrays: true // 保留没有匹配的订单文档
}
},
{
$match : {
"report.字段a":"xxx",
$or: [
{ "report.字段b": "aaa" }
]
}
},
{
$group: {
_id:null,
count:{$sum:1}
}
},
{
$project: {
count:1
}
}
])
例子2,关联查询list数据
db.getCollection("主表名").aggregate([
{
$lookup: {
from: "从表名", // 关联的集合名称
localField: "主表关联key", // 本地字段,orders 集合中用于关联的字段
foreignField: "从表关联key", // 外部字段,customers 集合中用于关联的字段
as: "report" // 关联结果存放的字段名
}
},
{
$unwind: {
path: "$report",
preserveNullAndEmptyArrays: true // 保留没有匹配的订单文档
}
},
{
$match : {
"report.字段a":"xxx",
$or: [
{ "report.字段b": "xxxc" }
]
}
},
{
$project: {
_id: 1,
主表字段a:1,
主表字段b:1,
report: {
_id: "$report._id",
......
}
}
},
{
$skip: 0 // 跳过的文档数,用于分页
},
{
$limit: 10 // 限制返回的文档数,用于分页
}
])
6.其他
6.1 可视化工具,MongoDB Compass,实用又好用,查看,编辑,导入导出,下载地址
https://www.mongodb.com/try/download/compass
6.2 navicat可以执行命令,查询数据选择行模式
6.3 springboot项目,打印命令行参数(参考),yml或properties中配置
logging.level.root debug
logging.level.org.springframework.data.mongodb.core: debug
6.4分片集群
分片集群部署
https://blog.csdn.net/github_38616039/article/details/134158118
分配数据创建
sh.enableSharding("库名")
db.getCollection(表名).createIndex({ fieldName: 1 })
sh.shardCollection('库名.表名',{pkValue:1})
db.adminCommand("flushRouterConfig")
sh.enableBalancing("库名.表名")
sh.startBalancer()
sh.status({"verbose":1})
db.表名.getShardDistribution()
7.参考网站
http://www.manongjc.com/detail/33-nlyyuqqbufycepj.html
https://www.mongodb.com/docs/manual/reference/operator/aggregation/count/
https://www.mongodb.com/docs/manual/reference/operator/aggregation/lookup/?_ga=2.167579261.2108825086.1710915715-66292563.1709890069
https://www.jb51.net/program/285486kdm.htm#_lab2_2_5
https://blog.csdn.net/superatom01/article/details/135004991?spm=1001.2014.3001.5506
https://www.knowledgedict.com/tutorial/mongodb-create-index.html