MongoDB错误记录及性能优化

1、该错误导致数据库启动后，几分钟（1小时内）继续崩溃停掉：

2020-01-23T12:56:17.302+0800 E STORAGE  [conn26] WiredTiger error (0) [1579755377:302619][16920:0x7f41128d2700], file:collection-501-1065032266366518290.wt, WT_CURSOR.search: read checksum error for 12288B block at offset 15403827200: block header checksum of 3096179896 doesn't match expected checksum of 2568315254
2020-01-23T12:56:17.302+0800 E STORAGE  [conn26] WiredTiger error (0) [1579755377:302699][16920:0x7f41128d2700], file:collection-501-1065032266366518290.wt, WT_CURSOR.search: collection-501-1065032266366518290.wt: encountered an illegal file format or internal value
2020-01-23T12:56:17.302+0800 E STORAGE  [conn26] WiredTiger error (-31804) [1579755377:302713][16920:0x7f41128d2700], file:collection-501-1065032266366518290.wt, WT_CURSOR.search: the process must exit and restart: WT_PANIC: WiredTiger library panic
2020-01-23T12:56:17.302+0800 I -        [conn26] Fatal Assertion 28558 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 361
2020-01-23T12:56:17.302+0800 I -        [conn4] Fatal Assertion 28559 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 64
2020-01-23T12:56:17.302+0800 I -        [conn18] Fatal Assertion 28559 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 64
2020-01-23T12:56:17.302+0800 I -        [conn15] Fatal Assertion 28559 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 64
2020-01-23T12:56:17.302+0800 I -        [conn26] 

***aborting after fassert() failure


2020-01-23T12:56:17.302+0800 I -        [conn4] 

***aborting after fassert() failure


2020-01-23T12:56:17.302+0800 I -        [conn18] 

***aborting after fassert() failure


2020-01-23T12:56:17.302+0800 I -        [conn15] 

***aborting after fassert() failure


2020-01-23T12:56:17.304+0800 I -        [conn25] Fatal Assertion 28559 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 64
2020-01-23T12:56:17.304+0800 I -        [conn25] 

***aborting after fassert() failure


2020-01-23T12:56:17.329+0800 F -        [conn26] Got signal: 6 (Aborted).

 0x7f4120bc2da1 0x7f4120bc1e99 0x7f4120bc237d 0x7f411e2105d0 0x7f411de6a2c7 0x7f411de6b9b8 0x7f411fe4ad45 0x7f41208d1446 0x7f411fe55028 0x7f411fe5511c 0x7f411fe55374 0x7f41214c8575 0x7f41214e5d40 0x7f41214ea168 0x7f412150a325 0x7f41214d5dd9 0x7f4121529c5e 0x7f41208c452e 0x7f412021f00b 0x7f41201c5a06 0x7f41201e77f3 0x7f41201c1e83 0x7f41201e77f3 0x7f41204ea7ba 0x7f41204eb0db 0x7f41204eb20d 0x7f412034aafe 0x7f41201728cf 0x7f412016e07c 0x7f41200a98e4 0x7f41200aa8c0 0x7f41206c65ad 0x7f41202c7432 0x7f41202c93a6 0x7f411feb954d 0x7f411feb9e8d 0x7f4120b28292 0x7f411e208dd5 0x7f411df3202d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"7F411F639000","o":"1589DA1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"7F411F639000","o":"1588E99"},{"b":"7F411F639000","o":"158937D"},{"b":"7F411E201000","o":"F5D0"},{"b":"7F411DE34000","o":"362C7","s":"gsignal"},{"b":"7F411DE34000","o":"379B8","s":"abort"},{"b":"7F411F639000","o":"811D45","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"7F411F639000","o":"1298446"},{"b":"7F411F639000","o":"81C028"},{"b":"7F411F639000","o":"81C11C","s":"__wt_err"},{"b":"7F411F639000","o":"81C374","s":"__wt_panic"},{"b":"7F411F639000","o":"1E8F575"},{"b":"7F411F639000","o":"1EACD40"},{"b":"7F411F639000","o":"1EB1168"},{"b":"7F411F639000","o":"1ED1325"},{"b":"7F411F639000","o":"1E9CDD9"},{"b":"7F411F639000","o":"1EF0C5E"},{"b":"7F411F639000","o":"128B52E","s":"_ZN5mongo21WiredTigerRecordStore6Cursor9seekExactERKNS_8RecordIdE"},{"b":"7F411F639000","o":"BE600B","s":"_ZN5mongo16WorkingSetCommon5fetchEPNS_16OperationContextEPNS_10WorkingSetEmNS_11unowned_ptrINS_20SeekableRecordCursorEEE"},{"b":"7F411F639000","o":"B8CA06","s":"_ZN5mongo10FetchStage6doWorkEPm"},{"b":"7F411F639000","o":"BAE7F3","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"7F411F639000","o":"B88E83","s":"_ZN5mongo11DeleteStage6doWorkEPm"},{"b":"7F411F639000","o":"BAE7F3","s":"_ZN5mongo9PlanStage4workEPm"},{"b":"7F411F639000","o":"EB17BA","s":"_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"7F411F639000","o":"EB20DB","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"7F411F639000","o":"EB220D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"7F411F639000","o":"D11AFE","s":"_ZN5mongo14performDeletesEPNS_16OperationContextERKNS_8DeleteOpE"},{"b":"7F411F639000","o":"B398CF","s":"_ZN5mongo9CmdDelete7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE"},{"b":"7F411F639000","o":"B3507C"},{"b":"7F411F639000","o":"A708E4","s":"_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE"},{"b":"7F411F639000","o":"A718C0","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE"},{"b":"7F411F639000","o":"108D5AD","s":"_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE"},{"b":"7F411F639000","o":"C8E432"},{"b":"7F411F639000","o":"C903A6","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"7F411F639000","o":"88054D","s":"_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE"},{"b":"7F411F639000","o":"880E8D"},{"b":"7F411F639000","o":"14EF292"},{"b":"7F411E201000","o":"7DD5"},{"b":"7F411DE34000","o":"FE02D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.4.2", "gitVersion" : "3f76e40c105fc223b3e5aac3e20dcd026b83b38b", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "3.10.0-514.el7.x86_64", "version" : "#1 SMP Tue Nov 22 16:42:41 UTC 2016", "machine" : "x86_64" }, "somap" : [ { "b" : "7F411F639000", "elfType" : 3, "buildId" : "BE957C997DB3613D354BB2EFE59D73F3E89250D8" }, { "b" : "7FFFA009C000", "elfType" : 3, "buildId" : "BF6B0C931C67B5BB2B0E80E07CBF73BAA5A466D2" }, { "b" : "7F411F1A3000", "path" : "/lib64/libssl.so.10", "elfType" : 3, "buildId" : "AEF5E6F2240B55F90E9DF76CFBB8B9D9F5286583" }, { "b" : "7F411ED41000", "path" : "/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "3593FA778645A59EA272DBBB59D318C60940E792" }, { "b" : "7F411EB39000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "EFDE2029C9A4A20BE5B8D8AE7E6551FF9B5755D2" }, { "b" : "7F411E935000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "357693C8F1F49D93010C4E31529C07CDD2BD3D08" }, { "b" : "7F411E633000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "5B14BE4D749631673523A61074C10959D50F5455" }, { "b" : "7F411E41D000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "179F202998E429AA1215907F6D4C5C1BB9C90136" }, { "b" : "7F411E201000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "96900CB0FF25B26F2BBDF247DE1408242E4773D8" }, { "b" : "7F411DE34000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "426A04647352308628F2091A30D347EDEEDED787" }, { "b" : "7F411F415000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "A527FE72908703C5972AE384E78D1850D1881EE7" }, { "b" : "7F411DBE7000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "BCC30853830CD911E58700591830DF51ABCBD7BA" }, { "b" : "7F411D8FE000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "45BAB0BB455BDFA960FDA22E4124CF17B67CC930" }, { "b" : "7F411D6FA000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "B4BE1023D9606A88169DF411BF94AF417D7BA1A0" }, { "b" : "7F411D4C7000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "A9B3906192687CC45D483AE3C58C8AF745A6726A" }, { "b" : "7F411D2B1000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "B9D5F73428BD6AD68C96986B57BEA3B7CEDB9745" }, { "b" : "7F411D0A1000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "94B3BCB669126166B77CDCE6092679A6AA2004C8" }, { "b" : "7F411CE9D000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "2E01D5AC08C1280D013AAB96B292AC58BC30A263" }, { "b" : "7F411CC84000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C444AE61E7CBB716FD9C18A0B46A7FE8F4FCF3E5" }, { "b" : "7F411CA5D000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "D2DD4DA3FDE1477D25BFFF80F3A25FDB541A8179" }, { "b" : "7F411C7FB000", "path" : "/lib64/libpcre.so.1", "elfType" : 3, "buildId" : "9CA3D11F018BEEB719CDB34BE800BF1641350D0A" } ] }}
 mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x7f4120bc2da1]
 mongod(+0x1588E99) [0x7f4120bc1e99]
 mongod(+0x158937D) [0x7f4120bc237d]
 libpthread.so.0(+0xF5D0) [0x7f411e2105d0]
 libc.so.6(gsignal+0x37) [0x7f411de6a2c7]
 libc.so.6(abort+0x148) [0x7f411de6b9b8]
 mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x7f411fe4ad45]
 mongod(+0x1298446) [0x7f41208d1446]
 mongod(+0x81C028) [0x7f411fe55028]
 mongod(__wt_err+0x9D) [0x7f411fe5511c]
 mongod(__wt_panic+0x24) [0x7f411fe55374]
 mongod(+0x1E8F575) [0x7f41214c8575]
 mongod(+0x1EACD40) [0x7f41214e5d40]
 mongod(+0x1EB1168) [0x7f41214ea168]
 mongod(+0x1ED1325) [0x7f412150a325]
 mongod(+0x1E9CDD9) [0x7f41214d5dd9]
 mongod(+0x1EF0C5E) [0x7f4121529c5e]
 mongod(_ZN5mongo21WiredTigerRecordStore6Cursor9seekExactERKNS_8RecordIdE+0x4E) [0x7f41208c452e]
 mongod(_ZN5mongo16WorkingSetCommon5fetchEPNS_16OperationContextEPNS_10WorkingSetEmNS_11unowned_ptrINS_20SeekableRecordCursorEEE+0xAB) [0x7f412021f00b]
 mongod(_ZN5mongo10FetchStage6doWorkEPm+0x106) [0x7f41201c5a06]
 mongod(_ZN5mongo9PlanStage4workEPm+0x63) [0x7f41201e77f3]
 mongod(_ZN5mongo11DeleteStage6doWorkEPm+0x363) [0x7f41201c1e83]
 mongod(_ZN5mongo9PlanStage4workEPm+0x63) [0x7f41201e77f3]
 mongod(_ZN5mongo12PlanExecutor11getNextImplEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x19A) [0x7f41204ea7ba]
 mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x4B) [0x7f41204eb0db]
 mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x6D) [0x7f41204eb20d]
 mongod(_ZN5mongo14performDeletesEPNS_16OperationContextERKNS_8DeleteOpE+0x59E) [0x7f412034aafe]
 mongod(_ZN5mongo9CmdDelete7runImplEPNS_16OperationContextERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKNS_7BSONObjERNS_14BSONObjBuilderE+0x4F) [0x7f41201728cf]
 mongod(+0xB3507C) [0x7f412016e07c]
 mongod(_ZN5mongo7Command3runEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS3_21ReplyBuilderInterfaceE+0x8E4) [0x7f41200a98e4]
 mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_RKNS_3rpc16RequestInterfaceEPNS4_21ReplyBuilderInterfaceE+0x9E0) [0x7f41200aa8c0]
 mongod(_ZN5mongo11runCommandsEPNS_16OperationContextERKNS_3rpc16RequestInterfaceEPNS2_21ReplyBuilderInterfaceE+0x26D) [0x7f41206c65ad]
 mongod(+0xC8E432) [0x7f41202c7432]
 mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x6D6) [0x7f41202c93a6]
 mongod(_ZN5mongo23ServiceEntryPointMongod12_sessionLoopERKSt10shared_ptrINS_9transport7SessionEE+0x1ED) [0x7f411feb954d]
 mongod(+0x880E8D) [0x7f411feb9e8d]
 mongod(+0x14EF292) [0x7f4120b28292]
 libpthread.so.0(+0x7DD5) [0x7f411e208dd5]
 libc.so.6(clone+0x6D) [0x7f411df3202d]
-----  END BACKTRACE  -----

解决办法：

/home/mongodb/bin/mongod --config /home/mongodb/conf/mongodb.conf --repair
结束后启动
/home/mongodb/bin/mongod --config /home/mongodb/conf/mongodb.conf

观察中…

可以用monit监控mongodb，崩溃后自动重启mongdb

2、log日志太大

use admin
db.auth('xxx','xxx')
db.runCommand({logRotate: 1})

错误

{
    "ok" : 0.0,
    "errmsg" : "not authorized on admin to execute command { logRotate: 1.0 }",
    "code" : 13,
    "codeName" : "Unauthorized"
}

缺少授权

> db.grantRolesToUser('admin', [{role:'clusterAdmin',db:'admin'}])
> db.runCommand({logRotate: 1})
{ "ok" : 1 }

3、CPU太高

查看数据库当前正在执行的操作

db.currentOp()

1、根据查询条件增加索引

2、大量数据排序会导致慢请求

当查询请求里包含排序的时候，如果排序无法通过索引满足，MongoDB 会在内存中对结果进行排序。
大家都知道，排序是非常消耗 CPU 的一项操作，最好在需要排序的字段上建立索引。
system.profile 中的 SORT 关键字反映了查询需要排序