mongodb

When did stress test, got an error as:
[quote]Moped::Errors::OperationFailure (The operation: #<Moped::Protocol::Command
@length=83
@request_id=17125
@response_to=0
@op_code=2004
@flags=[]
@full_collection_name="sf_core_development.$cmd"
@skip=0
@limit=-1
@selector={:getlasterror=>1, :w=>1}
@fields=nil>
[color=blue]failed with error 13636: "file /var/lib/mongodb/sf_core_development.5 open/create failed in createPrivateMap (look in log for more information)"
[/color]
See https://github.com/mongodb/mongo/blob/master/docs/errors.md
for details about this error.):
app/controllers/main_stages_controller.rb:106:in `block in create'
app/controllers/main_stages_controller.rb:105:in `create'
[/quote]


after some digging, got info from [url]https://github.com/mongodb/mongo/blob/master/docs/errors.md[/url]
in this page, check 13636, redirected to another page, and the reason was that:

bool ok = VirtualProtect(reinterpret_cast<void*>(protectStart),
                protectSize,
                PAGE_WRITECOPY,
                &oldProtection);
            if (!ok) {
                DWORD dosError = GetLastError();

                if (dosError == ERROR_COMMITMENT_LIMIT) {
                    // System has run out of memory between physical RAM & page file, tell the user
                    BSONObjBuilder bb;

                    ProcessInfo p;
                    p.getExtraInfo(bb);

                    severe() << "MongoDB has exhausted the system memory capacity.";
                    severe() << "Current Memory Status: " << bb.obj().toString();
                }

                severe() << "VirtualProtect for " << mmf->filename()
                    << " chunk " << chunkno
                    << " failed with " << errnoWithDescription(dosError)
                    << " (chunk size is " << protectSize
                    << ", address is " << hex << protectStart << dec << ")"
                    << " in mongo::makeChunkWritable, terminating"
                    << endl;

                fassertFailed(16362);
            }

checking mongodb log /var/log/mongodb/mongod.log, the reason is out of memory
[quote]
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B6B4B9"},{"b":"400000","o":"B0BCB1"},{"b":"400000","o":"AF0F0F"},{"b":"400000","o":"AF0FBC"},{"b":"400000","o":"93D56E"},{"b":"400000","o":"93D7AC"},{"b":"400000","o":"924C50"},{"b":"400000","o":"956191"},{"b":"400000","o":"9565A0"},{"b":"400000","o":"956832"},{"b":"400000","o":"95AB4E"},{"b":"400000","o":"966354"},{"b":"400000","o":"95B349"},{"b":"400000","o":"95B4F9"},{"b":"400000","o":"52EACE"},{"b":"400000","o":"52EF2D"},{"b":"400000","o":"6CDD4E"},{"b":"400000","o":"6D1062"},{"b":"400000","o":"6D2D40"},{"b":"400000","o":"429CBD"},{"b":"400000","o":"B1F0AB"},{"b":"7FC9283A0000","o":"6B50"},{"b":"7FC926E12000","o":"DB95D"}],"processInfo":{ "mongodbVersion" : "3.0.2", "gitVersion" : "6201872043ecbbc0a4cc169b5482dcf385fc464f", "uname" : { "sysname" : "Linux", "release" : "2.6.32-042stab104.1", "version" : "#1 SMP Thu Jan 29 12:58:41 MSK 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "FE06CE34D0A8EA7A70E5E8682BFCDB31D065F341" }, { "b" : "7FFFDE2FE000", "elfType" : 3, "buildId" : "853D0228D0C34846A6356C33001B9BEA56DA0C59" }, { "b" : "7FC9283A0000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "FEF281218797AD6AE726DD5FCEDECADD9E9F51DC" }, { "b" : "7FC928140000", "path" : "/usr/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "AEE5F3A05E87AFA440FCF6352C568A0F08584119" }, { "b" : "7FC927D48000", "path" : "/usr/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "37084B8E55653C947BA6295814D850D6AA0C561D" }, { "b" : "7FC927B40000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "F58D5DE3E7A2989E915422BA4203FE53DBA449A0" }, { "b" : "7FC92793C000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "5D1CA3A3D93ED5B6C6462FFA03E787FDBE4013A3" }, { "b" : "7FC927635000", "path" : "/usr/lib/x86_64-linux-gnu/libstdc++.so.6", "elfType" : 3, "buildId" : "8711429397A5AF8B6269B867D830EDF6E0225B8D" }, { "b" : "7FC9273B3000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "7F58D6664571941C86B2D969701A572AD4D7BF1D" }, { "b" : "7FC92719D000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "F980B1188708F8D8B5C35D185444AF4CB939AA1E" }, { "b" : "7FC926E12000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "A745EBA2C16BA80AE1EF1A7A7B70740C2CF1B363" }, { "b" : "7FC9285BC000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "9B23F2A44CC8CA6175CBD8D64584B1C7EA5FD18C" }, { "b" : "7FC926BFB000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "1EFEB71FD4999C2307570D673A724EA4E1D85267" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x29) [0xf6b4b9]
mongod(_ZN5mongo10logContextEPKc+0xE1) [0xf0bcb1]
mongod(_ZN5mongo11msgassertedEiPKc+0xAF) [0xef0f0f]
mongod(+0xAF0FBC) [0xef0fbc]
mongod(_ZN5mongo17DurableMappedFile13finishOpeningEv+0x2FE) [0xd3d56e]
mongod(_ZN5mongo17DurableMappedFile6createERKSsRyb+0x6C) [0xd3d7ac]
mongod(_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib+0x100) [0xd24c50]
mongod(_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib+0x131) [0xd56191]
mongod(_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib+0xE0) [0xd565a0]
mongod(_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib+0x1F2) [0xd56832]
mongod(_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib+0x3E) [0xd5ab4e]
mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x1F4) [0xd66354]
mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x59) [0xd5b349]
mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd5b4f9]
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x4E) [0x92eace]
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8D) [0x92ef2d]
mongod(_ZN5mongo14checkAndInsertEPNS_16OperationContextERNS_6Client7ContextEPKcRNS_7BSONObjE+0x13E) [0xacdd4e]
mongod(_ZN5mongo14receivedInsertEPNS_16OperationContextERNS_7MessageERNS_5CurOpE+0x1132) [0xad1062]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x16B0) [0xad2d40]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xDD) [0x829cbd]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x34B) [0xf1f0ab]
libpthread.so.0(+0x6B50) [0x7fc9283a6b50]
libc.so.6(clone+0x6D) [0x7fc926eed95d]
----- END BACKTRACE -----

[/quote]