Menu
Mongo seems to have run out of memory!? What can I do?
I am not even sure it's actually a memory problem.
2016-08-11T14:23:26.875+0200 E CONTROL [conn2575] mmap private failed with out of memory. (64 bit build)
2016-08-11T14:23:26.875+0200 I - [conn2575] Assertion: 13636:file /data/db/project.6 open/create failed in createPrivateMap (look in log for more information)
2016-08-11T14:23:26.892+0200 I CONTROL [conn2575]
0xf9b632 0xf36699 0xf1a22e 0xf1a2dc 0xd56925 0xd56b5f 0xd3cf06 0xd6ff6d 0xd70380 0xd7061d 0xd74ec2 0xd80b34 0xd756ab 0xd75859 0x90a811 0x90b3ec 0xa3e5fc 0xa3f334 0xbe8012 0xbe832b 0xbe8a0d 0x9b1769 0x9b2fc5 0x9b35f5 0x9b62dc 0x9d9071 0x9da07c 0x9dad4b 0xbb544b 0xac3e4a 0x7fd9c5 0xf4af79 0x7f9122c60aa1 0x7f91217b293d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"400000","o":"B9B632","s":"_ZN5mongo15printStackTraceERSo"},{"b":"400000","o":"B36699","s":"_ZN5mongo10logContextEPKc"},{"b":"400000","o":"B1A22E","s":"_ZN5mongo11msgassertedEiPKc"},{"b":"400000","o":"B1A2DC"},{"b":"400000","o":"956925","s":"_ZN5mongo17DurableMappedFile13finishOpeningEv"},{"b":"400000","o":"956B5F","s":"_ZN5mongo17DurableMappedFile6createERKSsRyb"},{"b":"400000","o":"93CF06","s":"_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib"},{"b":"400000","o":"96FF6D","s":"_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib"},{"b":"400000","o":"970380","s":"_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib"},{"b":"400000","o":"97061D","s":"_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib"},{"b":"400000","o":"974EC2","s":"_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib"},{"b":"400000","o":"980B34","s":"_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib"},{"b":"400000","o":"9756AB","s":"_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"975859","s":"_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib"},{"b":"400000","o":"50A811","s":"_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"50B3EC","s":"_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb"},{"b":"400000","o":"63E5FC","s":"_ZN5mongo11UpdateStage8doInsertEv"},{"b":"400000","o":"63F334","s":"_ZN5mongo11UpdateStage4workEPm"},{"b":"400000","o":"7E8012","s":"_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE"},{"b":"400000","o":"7E832B","s":"_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE"},{"b":"400000","o":"7E8A0D","s":"_ZN5mongo12PlanExecutor11executePlanEv"},{"b":"400000","o":"5B1769","s":"_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE"},{"b":"400000","o":"5B2FC5","s":"_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE"},{"b":"400000","o":"5B35F5","s":"_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE"},{"b":"400000","o":"5B62DC","s":"_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5D9071","s":"_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DA07C","s":"_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb"},{"b":"400000","o":"5DAD4B","s":"_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi"},{"b":"400000","o":"7B544B","s":"_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_"},{"b":"400000","o":"6C3E4A","s":"_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE"},{"b":"400000","o":"3FD9C5","s":"_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE"},{"b":"400000","o":"B4AF79","s":"_ZN5mongo17PortMessageServer17handleIncomingMsgEPv"},{"b":"7F9122C59000","o":"7AA1"},{"b":"7F91216CA000","o":"E893D","s":"clone"}],"processInfo":{ "mongodbVersion" : "3.0.9", "gitVersion" : "20d60d3491908f1ae252fe452300de3978a040c7", "uname" : { "sysname" : "Linux", "release" : "2.6.32-042stab113.11", "version" : "#1 SMP Fri Dec 18 17:32:04 MSK 2015", "machine" : "x86_64" }, "somap" : [ { "elfType" : 2, "b" : "400000", "buildId" : "47DFD0896283C6BF009395E08B673D3246CC7AE5" }, { "b" : "7FFD9BE1E000", "elfType" : 3, "buildId" : "F466F3FAB4F73E9911206E2486A3C5F8FCCA8EA5" }, { "b" : "7F9122C59000", "path" : "/lib64/libpthread.so.0", "elfType" : 3, "buildId" : "C56DD1B811FC0D9263248EBB308C73FCBCD80FC1" }, { "b" : "7F91229ED000", "path" : "/usr/lib64/libssl.so.10", "elfType" : 3, "buildId" : "24A4E7055CABC29C2DDAC730D635A20C08687AC6" }, { "b" : "7F912260A000", "path" : "/usr/lib64/libcrypto.so.10", "elfType" : 3, "buildId" : "F6F5BB27A852F29A134680794DD2EB5C3CC5D88B" }, { "b" : "7F9122402000", "path" : "/lib64/librt.so.1", "elfType" : 3, "buildId" : "95159178F1A4A3DBDC7819FBEA2C80E5FCDD6BAC" }, { "b" : "7F91221FE000", "path" : "/lib64/libdl.so.2", "elfType" : 3, "buildId" : "29B61382141595ECBA6576232E44F2310C3AAB72" }, { "b" : "7F9121EF8000", "path" : "/usr/lib64/libstdc++.so.6", "elfType" : 3, "buildId" : "C03877A9EE01DDC572E2B0F55F64C757773CF8D6" }, { "b" : "7F9121C74000", "path" : "/lib64/libm.so.6", "elfType" : 3, "buildId" : "989FE3A42CA8CEBDCC185A743896F23A0CF537ED" }, { "b" : "7F9121A5E000", "path" : "/lib64/libgcc_s.so.1", "elfType" : 3, "buildId" : "9350579A4970FA47F3144AD8F40B183B0954497D" }, { "b" : "7F91216CA000", "path" : "/lib64/libc.so.6", "elfType" : 3, "buildId" : "8E6FA4C4B0594C355C1B90C1D49990368C81A040" }, { "b" : "7F9122E76000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "959C5E10A47EE8A633E7681B64B4B9F74E242ED5" }, { "b" : "7F9121486000", "path" : "/lib64/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "441FA45097A11508E50D55A3D1FF169BF2BE7C62" }, { "b" : "7F912119F000", "path" : "/lib64/libkrb5.so.3", "elfType" : 3, "buildId" : "F62622218875795666E08B92D176A50791183EEC" }, { "b" : "7F9120F9B000", "path" : "/lib64/libcom_err.so.2", "elfType" : 3, "buildId" : "152E2C18A7A2145021A8A879A01A82EE134E3946" }, { "b" : "7F9120D6F000", "path" : "/lib64/libk5crypto.so.3", "elfType" : 3, "buildId" : "B8DEDADC140347276164C729418C7A37B7224135" }, { "b" : "7F9120B59000", "path" : "/lib64/libz.so.1", "elfType" : 3, "buildId" : "5FA8E5038EC04A774AF72A9BB62DC86E1049C4D6" }, { "b" : "7F912094E000", "path" : "/lib64/libkrb5support.so.0", "elfType" : 3, "buildId" : "4BDFC7A19C1F328EB4FCFBCE7A1E27606928610D" }, { "b" : "7F912074B000", "path" : "/lib64/libkeyutils.so.1", "elfType" : 3, "buildId" : "AF374BAFB7F5B139A0B431D3F06D82014AFF3251" }, { "b" : "7F9120531000", "path" : "/lib64/libresolv.so.2", "elfType" : 3, "buildId" : "C39D7FFB49DFB1B55AD09D1D711AD802123F6623" }, { "b" : "7F9120312000", "path" : "/lib64/libselinux.so.1", "elfType" : 3, "buildId" : "E6798A06BEE17CF102BBA44FD512FF8B805CEAF1" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x32) [0xf9b632]
mongod(_ZN5mongo10logContextEPKc+0xE9) [0xf36699]
mongod(_ZN5mongo11msgassertedEiPKc+0xAE) [0xf1a22e]
mongod(+0xB1A2DC) [0xf1a2dc]
mongod(_ZN5mongo17DurableMappedFile13finishOpeningEv+0x2F5) [0xd56925]
mongod(_ZN5mongo17DurableMappedFile6createERKSsRyb+0x6F) [0xd56b5f]
mongod(_ZN5mongo8DataFile4openEPNS_16OperationContextEPKcib+0x106) [0xd3cf06]
mongod(_ZN5mongo19MmapV1ExtentManager9_addAFileEPNS_16OperationContextEib+0x15D) [0xd6ff6d]
mongod(_ZN5mongo19MmapV1ExtentManager13_createExtentEPNS_16OperationContextEib+0xE0) [0xd70380]
mongod(_ZN5mongo19MmapV1ExtentManager14allocateExtentEPNS_16OperationContextEbib+0x1FD) [0xd7061d]
mongod(_ZN5mongo17RecordStoreV1Base19increaseStorageSizeEPNS_16OperationContextEib+0x42) [0xd74ec2]
mongod(_ZN5mongo19SimpleRecordStoreV111allocRecordEPNS_16OperationContextEib+0x1F4) [0xd80b34]
mongod(_ZN5mongo17RecordStoreV1Base13_insertRecordEPNS_16OperationContextEPKcib+0x5B) [0xd756ab]
mongod(_ZN5mongo17RecordStoreV1Base12insertRecordEPNS_16OperationContextEPKcib+0x79) [0xd75859]
mongod(_ZN5mongo10Collection15_insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x61) [0x90a811]
mongod(_ZN5mongo10Collection14insertDocumentEPNS_16OperationContextERKNS_7BSONObjEb+0x8C) [0x90b3ec]
mongod(_ZN5mongo11UpdateStage8doInsertEv+0x17C) [0xa3e5fc]
mongod(_ZN5mongo11UpdateStage4workEPm+0xD4) [0xa3f334]
mongod(_ZN5mongo12PlanExecutor18getNextSnapshottedEPNS_11SnapshottedINS_7BSONObjEEEPNS_8RecordIdE+0x142) [0xbe8012]
mongod(_ZN5mongo12PlanExecutor7getNextEPNS_7BSONObjEPNS_8RecordIdE+0x3B) [0xbe832b]
mongod(_ZN5mongo12PlanExecutor11executePlanEv+0x3D) [0xbe8a0d]
mongod(_ZN5mongo18WriteBatchExecutor10execUpdateERKNS_12BatchItemRefEPNS_7BSONObjEPPNS_16WriteErrorDetailE+0x719) [0x9b1769]
mongod(_ZN5mongo18WriteBatchExecutor11bulkExecuteERKNS_21BatchedCommandRequestERKNS_19WriteConcernOptionsEPSt6vectorIPNS_19BatchedUpsertDetailESaIS9_EEPS7_IPNS_16WriteErrorDetailESaISE_EE+0x245) [0x9b2fc5]
mongod(_ZN5mongo18WriteBatchExecutor12executeBatchERKNS_21BatchedCommandRequestEPNS_22BatchedCommandResponseE+0x465) [0x9b35f5]
mongod(_ZN5mongo8WriteCmd3runEPNS_16OperationContextERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x17C) [0x9b62dc]
mongod(_ZN5mongo12_execCommandEPNS_16OperationContextEPNS_7CommandERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x31) [0x9d9071]
mongod(_ZN5mongo7Command11execCommandEPNS_16OperationContextEPS0_iPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0xC6C) [0x9da07c]
mongod(_ZN5mongo12_runCommandsEPNS_16OperationContextEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x28B) [0x9dad4b]
mongod(_ZN5mongo8runQueryEPNS_16OperationContextERNS_7MessageERNS_12QueryMessageERKNS_15NamespaceStringERNS_5CurOpES3_+0x73B) [0xbb544b]
mongod(_ZN5mongo16assembleResponseEPNS_16OperationContextERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0xABA) [0xac3e4a]
mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0xF5) [0x7fd9c5]
mongod(_ZN5mongo17PortMessageServer17handleIncomingMsgEPv+0x339) [0xf4af79]
libpthread.so.0(+0x7AA1) [0x7f9122c60aa1]
libc.so.6(clone+0x6D) [0x7f91217b293d]
----- END BACKTRACE -----
ulimit -a
core file size (blocks, -c) 0
data seg size (kbytes, -d) unlimited
scheduling priority (-e) 0
file size (blocks, -f) unlimited
pending signals (-i) 127948
max locked memory (kbytes, -l) 64
max memory size (kbytes, -m) unlimited
open files (-n) 1024
pipe size (512 bytes, -p) 8
POSIX message queues (bytes, -q) 819200
real-time priority (-r) 0
stack size (kbytes, -s) 10240
cpu time (seconds, -t) unlimited
max user processes (-u) 127948
virtual memory (kbytes, -v) unlimited
file locks (-x) unlimited
ulimit -v
unlimited
swapon -s
Filename Type Size Used Priority
/dev/null partition 524288 175344 -1
free -m
total used free shared buffers cached
Mem: 4096 3331 764 15 0 2985
-/+ buffers/cache: 345 3750
Swap: 512 171 340
df -h
Filesystem Size Used Avail Use% Mounted on
/dev/ploop10759p1 40G 26G 12G 69% /
none 16G 4.0K 16G 1% /dev
none 16G 0 16G 0% /dev/shm
# cat /proc/sys/vm/overcommit_memory
0
ps aux | grep mongo
webdev 4006 0.2 26.0 9123780 1091292 ? Sl 14:54 0:06 mongod --logpath /var/log/mongodb/mongod.log
webdev 6031 0.0 0.0 103312 872 pts/1 S+ 15:35 0:00 grep mongo
mongod --version
db version v3.0.12
git version: 33934938e0e95d534cebbaff656cde916b9c3573
cat /etc/centos-release
CentOS release 6.8 (Final)
db.serverStatus()
{
"host" : "host",
"version" : "3.0.12",
"process" : "mongod",
"pid" : NumberLong(4006),
"uptime" : 3590,
"uptimeMillis" : NumberLong(3590668),
"uptimeEstimate" : 3555,
"localTime" : ISODate("2016-08-11T13:54:51.869Z"),
"asserts" : {
"regular" : 0,
"warning" : 1,
"msg" : 0,
"user" : 0,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 59,
"total_ms" : 80,
"average_ms" : 1.3559322033898304,
"last_ms" : 1,
"last_finished" : ISODate("2016-08-11T13:54:01.250Z")
},
"connections" : {
"current" : 4,
"available" : 815,
"totalCreated" : NumberLong(6)
},
"cursors" : {
"note" : "deprecated, use server status metrics",
"clientCursors_size" : 0,
"totalOpen" : 0,
"pinned" : 0,
"totalNoTimeout" : 0,
"timedOut" : 0
},
"dur" : {
"commits" : 24,
"journaledMB" : 0,
"writeToDataFilesMB" : 0,
"compression" : 0,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 0,
"prepLogBuffer" : 0,
"writeToJournal" : 0,
"writeToDataFiles" : 0,
"remapPrivateView" : 0,
"commits" : 0,
"commitsInWriteLock" : 0
}
},
"extra_info" : {
"note" : "fields vary by platform",
"heap_usage_bytes" : 63466248,
"page_faults" : 28
},
"globalLock" : {
"totalTime" : NumberLong("3590669000"),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 12,
"readers" : 0,
"writers" : 0
}
},
"locks" : {
"Global" : {
"acquireCount" : {
"r" : NumberLong(47395),
"w" : NumberLong(1270),
"W" : NumberLong(5)
},
"acquireWaitCount" : {
"r" : NumberLong(1)
},
"timeAcquiringMicros" : {
"r" : NumberLong(641)
}
},
"MMAPV1Journal" : {
"acquireCount" : {
"r" : NumberLong(23058),
"w" : NumberLong(1300),
"R" : NumberLong(35093)
},
"acquireWaitCount" : {
"w" : NumberLong(3),
"R" : NumberLong(3)
},
"timeAcquiringMicros" : {
"w" : NumberLong(24),
"R" : NumberLong(63215)
}
},
"Database" : {
"acquireCount" : {
"r" : NumberLong(23056),
"w" : NumberLong(1247),
"R" : NumberLong(4),
"W" : NumberLong(23)
}
},
"Collection" : {
"acquireCount" : {
"R" : NumberLong(24059),
"W" : NumberLong(1247)
}
},
"Metadata" : {
"acquireCount" : {
"W" : NumberLong(1)
}
}
},
"network" : {
"bytesIn" : NumberLong(28574),
"bytesOut" : NumberLong(14793714),
"numRequests" : NumberLong(229)
},
"opcounters" : {
"insert" : 0,
"query" : 216,
"update" : 1,
"delete" : 0,
"getmore" : 3,
"command" : 11
},
"opcountersRepl" : {
"insert" : 0,
"query" : 0,
"update" : 0,
"delete" : 0,
"getmore" : 0,
"command" : 0
},
"storageEngine" : {
"name" : "mmapv1"
},
"writeBacksQueued" : false,
"mem" : {
"bits" : 64,
"resident" : 1068,
"virtual" : 8909,
"supported" : true,
"mapped" : 4287,
"mappedWithJournal" : 8574
},
"metrics" : {
"commands" : {
"getLog" : {
"failed" : NumberLong(0),
"total" : NumberLong(1)
},
"isMaster" : {
"failed" : NumberLong(0),
"total" : NumberLong(5)
},
"replSetGetStatus" : {
"failed" : NumberLong(1),
"total" : NumberLong(1)
},
"serverStatus" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
},
"update" : {
"failed" : NumberLong(0),
"total" : NumberLong(1)
},
"whatsmyuri" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
}
},
"cursor" : {
"timedOut" : NumberLong(0),
"open" : {
"noTimeout" : NumberLong(0),
"pinned" : NumberLong(0),
"total" : NumberLong(0)
}
},
"document" : {
"deleted" : NumberLong(0),
"inserted" : NumberLong(0),
"returned" : NumberLong(26915),
"updated" : NumberLong(0)
},
"getLastError" : {
"wtime" : {
"num" : 0,
"totalMillis" : 0
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"fastmod" : NumberLong(0),
"idhack" : NumberLong(0),
"scanAndOrder" : NumberLong(140),
"writeConflicts" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(599),
"scannedObjects" : NumberLong(439045)
},
"record" : {
"moves" : NumberLong(0)
},
"repl" : {
"apply" : {
"batches" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : 268435456,
"sizeBytes" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(0),
"getmores" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0),
"readersCreated" : NumberLong(0)
},
"preload" : {
"docs" : {
"num" : 0,
"totalMillis" : 0
},
"indexes" : {
"num" : 0,
"totalMillis" : 0
}
}
},
"storage" : {
"freelist" : {
"search" : {
"bucketExhausted" : NumberLong(0),
"requests" : NumberLong(1),
"scanned" : NumberLong(0)
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(0),
"passes" : NumberLong(59)
}
},
"ok" : 1
}
890
Looks like you're currently capped at 1024 open files. Although there's usually a boost error to accompany that kind of failure, it's worth trying something like ulimit -n 4096
see the docs: https://docs.mongodb.com/manual/reference/ulimit/
168
If you love us? You can donate to us via Paypal or buy me a coffee so we can maintain and grow! Thank you!
Donate Us With
