备注:
MongoDB 4.2 版本
文章目录
测试数据准备
我用shell写了个程序,往mongodb里面插入一个100w的数据
[root@10-31-1-126 shell]# more batch_testdat.sh
#! /bin/bash
ins_mongo()
{
mongo 127.0.0.1:27017/zqs <<EOF
db.auth("zqs_user","123456")
for (var i = 1; i <= 1000000; i++) db.t1.save({id : i, name : 'aaa',date: new Date()});
EOF
}
ins_mongo
运行shell脚本:
-- 这个运行比较慢
sh batch_testdat.sh >/dev/null &
一.数据导入和导出
mongodb提供了导入和导出的命令:
mongoexport和mongoimport
1.1 mongoexport
1.1.1 查看mongo下面有什么数据
[root@10-31-1-126 ~]# mongo
MongoDB shell version v4.2.10
connecting to: mongodb://127.0.0.1:27017/?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("138b0e26-258c-47bb-ada2-620d1422403b") }
MongoDB server version: 4.2.10
rs:PRIMARY> use admin
switched to db admin
rs:PRIMARY> db.auth("admin","123456")
1
rs:PRIMARY> show dbs
admin 0.000GB
config 0.000GB
local 0.027GB
test 0.000GB
zqs 0.025GB
1.1.2 导出collection为t1的数据
主要参数:
-d:要导出的库
-c:要导出的表
-o:导出的文件名
-q:查询条件
-f:导出哪几列
其他参数详情使用 mongoexport --help 查看详情
-- 导出zqs下t1表 类别为csv,指定3列
mongoexport -u zqs_user -p 123456 -d zqs -c t1 -f id,name,date --type=csv -o t1.csv
测试记录:
[root@10-31-1-126 backup]# mongoexport -u zqs_user -p 123456 -d zqs -c t1 -f id,name,date --type=csv -o t1.csv
2020-11-11T09:22:15.566+0800 connected to: mongodb://localhost/
2020-11-11T09:22:16.567+0800 [##......................] zqs.t1 96000/1000000 (9.6%)
2020-11-11T09:22:17.567+0800 [#####...................] zqs.t1 240000/1000000 (24.0%)
2020-11-11T09:22:18.567+0800 [########................] zqs.t1 360000/1000000 (36.0%)
2020-11-11T09:22:19.567+0800 [###########.............] zqs.t1 480000/1000000 (48.0%)
2020-11-11T09:22:20.567+0800 [##############..........] zqs.t1 608000/1000000 (60.8%)
2020-11-11T09:22:21.567+0800 [#################.......] zqs.t1 720000/1000000 (72.0%)
2020-11-11T09:22:22.567+0800 [####################....] zqs.t1 864000/1000000 (86.4%)
2020-11-11T09:22:23.556+0800 [########################] zqs.t1 1000000/1000000 (100.0%)
2020-11-11T09:22:23.556+0800 exported 1000000 records
[root@10-31-1-126 backup]#
[root@10-31-1-126 backup]#
[root@10-31-1-126 backup]# more t1.csv
id,name,date
1,aaa,2020-11-10T11:56:28.306Z
2,aaa,2020-11-10T11:56:28.311Z
3,aaa,2020-11-10T11:56:28.312Z
4,aaa,2020-11-10T11:56:28.313Z
5,aaa,2020-11-10T11:56:28.314Z
6,aaa,2020-11-10T11:56:28.314Z
7,aaa,2020-11-10T11:56:28.315Z
8,aaa,2020-11-10T11:56:28.316Z
9,aaa,2020-11-10T11:56:28.316Z
10,aaa,2020-11-10T11:56:28.317Z
11,aaa,2020-11-10T11:56:28.318Z
12,aaa,2020-11-10T11:56:28.318Z
13,aaa,2020-11-10T11:56:28.319Z
14,aaa,2020-11-10T11:56:28.320Z
15,aaa,2020-11-10T11:56:28.320Z
16,aaa,2020-11-10T11:56:28.321Z
17,aaa,2020-11-10T11:56:28.321Z
18,aaa,2020-11-10T11:56:28.322Z
19,aaa,2020-11-10T11:56:28.323Z
20,aaa,2020-11-10T11:56:28.323Z
21,aaa,2020-11-10T11:56:28.324Z
22,aaa,2020-11-10T11:56:28.324Z
23,aaa,2020-11-10T11:56:28.325Z
24,aaa,2020-11-10T11:56:28.325Z
1.2 mongoimport
mongoimport可以把特定格式文件(JSON、CVS)中的内容导出到collection中。
主要参数:
-f:导出哪几列
–headerline:将第一行作为表头(只支持CSV和TSV格式)
–fields 和 --headerline 不兼容。
-- 将t1表转为t2表
mongoimport -u zqs_user -p 123456 -d zqs -c t2 --type=csv --headerline --file=t1.csv
测试记录:
[root@10-31-1-126 backup]# mongoimport -u zqs_user -p 123456 -d zqs -c t2 --type=csv --headerline --file=t1.csv
2020-11-11T09:32:34.643+0800 connected to: mongodb://localhost/
2020-11-11T09:32:37.643+0800 [##......................] zqs.t2 3.10MB/34.2MB (9.1%)
2020-11-11T09:32:40.643+0800 [####....................] zqs.t2 6.35MB/34.2MB (18.6%)
2020-11-11T09:32:43.643+0800 [######..................] zqs.t2 9.75MB/34.2MB (28.5%)
2020-11-11T09:32:46.643+0800 [#########...............] zqs.t2 13.2MB/34.2MB (38.4%)
2020-11-11T09:32:49.643+0800 [###########.............] zqs.t2 16.5MB/34.2MB (48.2%)
2020-11-11T09:32:52.643+0800 [##############..........] zqs.t2 20.2MB/34.2MB (59.0%)
2020-11-11T09:32:55.643+0800 [################........] zqs.t2 23.1MB/34.2MB (67.6%)
2020-11-11T09:32:58.643+0800 [##################......] zqs.t2 26.6MB/34.2MB (77.6%)
2020-11-11T09:33:01.643+0800 [#####################...] zqs.t2 30.2MB/34.2MB (88.1%)
2020-11-11T09:33:04.643+0800 [#######################.] zqs.t2 33.5MB/34.2MB (97.9%)
2020-11-11T09:33:05.625+0800 [########################] zqs.t2 34.2MB/34.2MB (100.0%)
2020-11-11T09:33:05.626+0800 1000000 document(s) imported successfully. 0 document(s) failed to import.
[root@10-31-1-126 backup]#
二.数据备份和恢复
MongoDB将所有数据存储在数据目录中,默认/data/db,可以将所有数据目录下的文件拷贝出来用于创建备份。但是对运行中的MongoDB,采用拷贝文件的方式创建备份可能会造成文件的损坏。为了在不影响业务正常运行下进行备份,我们可以使用MongoDB提供的备份和恢复工具:mongodump和mongorestore。
2.1 mongodump
2.1.1 使用fsync命令强制MongoDB服务器同步所有内存数据,然后对数据库加锁防止写入操作
[root@10-31-1-126 backup]# mongo
MongoDB shell version v4.2.10
connecting to: mongodb://127.0.0.1:27017/?compressors=disabled&gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("79f4ee82-4756-45db-8c9e-0d796eec5826") }
MongoDB server version: 4.2.10
rs:PRIMARY>
rs:PRIMARY>
rs:PRIMARY> use admin
switched to db admin
rs:PRIMARY> db.auth("root","123456")
1
rs:PRIMARY> db.runCommand({"fsync":1,"lock":1});
{
"info" : "now locked against writes, use db.fsyncUnlock() to unlock",
"lockCount" : NumberLong(1),
"seeAlso" : "http://dochub.mongodb.org/core/fsynccommand",
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1605059519, 1),
"signature" : {
"hash" : BinData(0,"+lsB/7qVst6tKVp6RCPcO5AyqaM="),
"keyId" : NumberLong("6893333011551485956")
}
},
"operationTime" : Timestamp(1605059519, 1)
}
rs:PRIMARY>
2.1.2 mongodump备份
进行数据备份操作。默认保存到 ./dump 目录下
主要参数:
-d:要导出的库
-c:要导出的表
-o:导出的文件名
-q:查询条件
[root@10-31-1-126 backup]# mongodump -u zqs_user -p 123456 -d zqs -c t1
2020-11-11T09:54:56.539+0800 writing zqs.t1 to dump/zqs/t1.bson
2020-11-11T09:54:57.669+0800 done dumping zqs.t1 (1000000 documents)
-- 备份到了当前目录下的 dump/zqs/ 目录下
[root@10-31-1-126 backup]# cd ./dump/zqs/
[root@10-31-1-126 zqs]# ls -lrth
总用量 60M
-rw-r--r--. 1 root root 154 11月 11 09:54 t1.metadata.json
-rw-r--r--. 1 root root 60M 11月 11 09:54 t1.bson
2.1.3 解锁
对数据解锁,允许数据写入
rs:PRIMARY> db.fsyncUnlock();
{
"info" : "fsyncUnlock completed",
"lockCount" : NumberLong(0),
"ok" : 1,
"$clusterTime" : {
"clusterTime" : Timestamp(1605059519, 1),
"signature" : {
"hash" : BinData(0,"+lsB/7qVst6tKVp6RCPcO5AyqaM="),
"keyId" : NumberLong("6893333011551485956")
}
},
"operationTime" : Timestamp(1605059519, 1)
}
rs:PRIMARY>
2.2 mongorestore
2.2.1 删除t1表
rs:PRIMARY> use zqs
switched to db zqs
rs:PRIMARY> db.t1.drop()
true
2.2.2 恢复刚刚备份出来的t1表
主要参数:
-d:要备份的库
-c:要备份的表
–drop:恢复备份前删除
[root@10-31-1-126 backup]# mongorestore -u zqs_user -p 123456 -d zqs -c t1 ./dump/zqs/t1.bson
2020-11-11T10:01:42.696+0800 checking for collection data in dump/zqs/t1.bson
2020-11-11T10:01:42.697+0800 reading metadata for zqs.t1 from dump/zqs/t1.metadata.json
2020-11-11T10:01:42.710+0800 restoring zqs.t1 from dump/zqs/t1.bson
2020-11-11T10:01:45.697+0800 [######..................] zqs.t1 16.9MB/59.1MB (28.6%)
2020-11-11T10:01:48.698+0800 [############............] zqs.t1 30.1MB/59.1MB (50.9%)
2020-11-11T10:01:51.697+0800 [#################.......] zqs.t1 42.6MB/59.1MB (72.0%)
2020-11-11T10:01:54.695+0800 [######################..] zqs.t1 55.8MB/59.1MB (94.4%)
2020-11-11T10:01:55.447+0800 [########################] zqs.t1 59.1MB/59.1MB (100.0%)
2020-11-11T10:01:55.447+0800 no indexes to restore
2020-11-11T10:01:55.447+0800 finished restoring zqs.t1 (1000000 documents, 0 failures)
2020-11-11T10:01:55.447+0800 1000000 document(s) restored successfully. 0 document(s) failed to restore.
[root@10-31-1-126 backup]#
三.克隆collection
克隆技术可以将数据从一个数据源拷贝到多个数据源,将一份数据发布到多个存储服务器上
官方文档显示,如果开启认证的mongodb是不能使用cloneCollection的
cloneCollection cannot be used if the from server has authorization enabled.
-- 克隆远程的
db.runCommand({cloneCollection:"zqs.t1",from:"10.31.1.126:27017"});
-- 克隆本地(简直不要太慢)
db.t1.find().forEach(function(x){db.t2.insert(x)});
测试记录:
rs:PRIMARY>
rs:PRIMARY> db.t1.find().forEach(function(x){db.t3.insert(x)});
rs:PRIMARY>
rs:PRIMARY> db.t3.count()
1000000
rs:PRIMARY> db.t3.find()
{ "_id" : ObjectId("5faa7fec5559ed33b60479ff"), "id" : 2, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.311Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a11"), "id" : 20, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.323Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a18"), "id" : 27, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.326Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a1a"), "id" : 29, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.327Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a1b"), "id" : 30, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.328Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a1d"), "id" : 32, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.329Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a22"), "id" : 37, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.331Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a4d"), "id" : 80, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.351Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a4e"), "id" : 81, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.351Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a4f"), "id" : 82, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.352Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a50"), "id" : 83, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.353Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a51"), "id" : 84, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.353Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a52"), "id" : 85, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.354Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a53"), "id" : 86, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.354Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a54"), "id" : 87, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.354Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a56"), "id" : 89, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.355Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a58"), "id" : 91, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.356Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a59"), "id" : 92, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.357Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a5a"), "id" : 93, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.359Z") }
{ "_id" : ObjectId("5faa7fec5559ed33b6047a5b"), "id" : 94, "name" : "aaa", "date" : ISODate("2020-11-10T11:56:28.359Z") }
Type "it" for more
rs:PRIMARY>
参考
1.https://www.jianshu.com/p/d65190e16afe