Database error MongoDB: disappeared database

Question

Launched a very large number of processes. Then rebooted the server and the database is gone. When I walked into rockmongo, I have not seen the old database. There was a new database admin and nothing more.

 db.serverStatus()
{
        "host" : "host",
        "version" : "2.0.6",
        "process" : "mongod",
        "uptime" : 1041,
        "uptimeEstimate" : 1037,
        "localTime" : ISODate("2012-07-26T08:20:40.729Z"),
        "globalLock" : {
                "totalTime" : 1041408230,
                "lockTime" : 330771,
                "ratio" : 0.0003176189610101314,
                "currentQueue" : {
                        "total" : 0,
                        "readers" : 0,
                        "writers" : 0
                },
                "activeClients" : {
                        "total" : 0,
                        "readers" : 0,
                        "writers" : 0
                }
        },
        "mem" : {
                "bits" : 64,
                "resident" : 19,
                "virtual" : 114,
                "supported" : true,
                "mapped" : 0,
                "mappedWithJournal" : 0
        },
        "connections" : {
                "current" : 11,
                "available" : 808
        },
        "extra_info" : {
                "note" : "fields vary by platform",
                "heap_usage_bytes" : 3623360,
                "page_faults" : 2
        },
        "indexCounters" : {
                "btree" : {
                        "accesses" : 0,
                        "hits" : 0,
                        "misses" : 0,
                        "resets" : 0,
                        "missRatio" : 0
                }
        },
        "backgroundFlushing" : {
                "flushes" : 17,
                "total_ms" : 0,
                "average_ms" : 0,
                "last_ms" : 0,
                "last_finished" : ISODate("2012-07-26T08:20:19.341Z")
        },
        "cursors" : {
                "totalOpen" : 0,
                "clientCursors_size" : 0,
                "timedOut" : 0
        },
        "network" : {
                "bytesIn" : 4429,
                "bytesOut" : 14192,
                "numRequests" : 56
        },
        "opcounters" : {
                "insert" : 0,
                "query" : 22,
                "update" : 0,
                "delete" : 0,
                "getmore" : 0,
                "command" : 54
        },
        "asserts" : {
                "regular" : 0,
                "warning" : 14,
                "msg" : 0,
                "user" : 0,
                "rollovers" : 0
        },
        "writeBacksQueued" : false,
        "dur" : {
                "commits" : 29,
                "journaledMB" : 0,
                "writeToDataFilesMB" : 0,
                "compression" : 0,
                "commitsInWriteLock" : 0,
                "earlyCommits" : 0,
                "timeMs" : {
                        "dt" : 3045,
                        "prepLogBuffer" : 0,
                        "writeToJournal" : 0,
                        "writeToDataFiles" : 0,
                        "remapPrivateView" : 0
                }
        },
        "ok" : 1
}

mongodb.log

    {
            "assertion" : "file /var/lib/mongodb/baza.4 open/create failed in create                                                                             PrivateMap (look in log for more information)",
            "assertionCode" : 13636,
            "errmsg" : "db assertion failure",
            "ok" : 0
    }

Thu Jul 26 12:05:44 [conn5] ERROR: mmap private failed with out of memory. (64 bit build)
Thu Jul 26 12:05:44 [conn5] Assertion: 13636:file /var/lib/mongodb/baza.4 open/create failed in createPrivateMap (look in log for more information)
0x5848a2 0x75dcea 0x75e8e3 0x8a8c3b 0x8a0962 0x8a1d03 0x8a1fda 0x8a2835 0x898619 0x898ea3 0x982bd5 0x97c4f4 0x97dbfd 0x940f05 0x943a30 0x889107 0x88dcd9 0xaa0bc8 0x6389f7 0x2b76122978ca 
 /usr/bin/mongod(_ZN5mongo11msgassertedEiPKc+0x112) [0x5848a2]
 /usr/bin/mongod(_ZN5mongo8MongoMMF13finishOpeningEv+0x1ca) [0x75dcea]
 /usr/bin/mongod(_ZN5mongo8MongoMMF6createESsRyb+0x63) [0x75e8e3]
 /usr/bin/mongod(_ZN5mongo13MongoDataFile4openEPKcib+0x14b) [0x8a8c3b]
 /usr/bin/mongod(_ZN5mongo8Database7getFileEiib+0x102) [0x8a0962]
 /usr/bin/mongod(_ZN5mongo8Database12openAllFilesEv+0x33) [0x8a1d03]
 /usr/bin/mongod(_ZN5mongo8DatabaseC1EPKcRbRKSs+0x1ca) [0x8a1fda]
 /usr/bin/mongod(_ZN5mongo14DatabaseHolder11getOrCreateERKSsS2_Rb+0x605) [0x8a2835]
 /usr/bin/mongod(_ZN5mongo6Client7Context11_finishInitEb+0x2e9) [0x898619]
 /usr/bin/mongod(_ZN5mongo6Client7ContextC1ERKSsSsPNS_9mongolockEb+0x73) [0x898ea3]
 /usr/bin/mongod(_ZN5mongo16CmdListDatabases3runERKSsRNS_7BSONObjEiRSsRNS_14BSONObjBuilderEb+0x215) [0x982bd5]
 /usr/bin/mongod(_ZN5mongo11execCommandEPNS_7CommandERNS_6ClientEiPKcRNS_7BSONObjERNS_14BSONObjBuilderEb+0x6c4) [0x97c4f4]
 /usr/bin/mongod(_ZN5mongo12_runCommandsEPKcRNS_7BSONObjERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x6fd) [0x97dbfd]
 /usr/bin/mongod(_ZN5mongo11runCommandsEPKcRNS_7BSONObjERNS_5CurOpERNS_11_BufBuilderINS_16TrivialAllocatorEEERNS_14BSONObjBuilderEbi+0x35) [0x940f05]
 /usr/bin/mongod(_ZN5mongo8runQueryERNS_7MessageERNS_12QueryMessageERNS_5CurOpES1_+0x1740) [0x943a30]
 /usr/bin/mongod() [0x889107]
 /usr/bin/mongod(_ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE+0x559) [0x88dcd9]
 /usr/bin/mongod(_ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE+0x78) [0xaa0bc8]
 /usr/bin/mongod(_ZN5mongo3pms9threadRunEPNS_13MessagingPortE+0x287) [0x6389f7]
 /lib/libpthread.so.0(+0x68ca) [0x2b76122978ca]
Thu Jul 26 12:05:52 [initandlisten] connection accepted from 127.0.0.1:41970 #11

~# free -ltm

             total       used       free     shared    buffers     cached
Mem:          1400        196       1203          0          0          0
Low:          1400        196       1203
High:            0          0          0
-/+ buffers/cache:        196       1203
Swap:            0          0          0
Total:        1400        196       1203

~# cat /proc/sys/vm/overcommit_memory

0

~# cat /proc/sys/vm/overcommit_ratio

50

~# ulimit -a

core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 16382
max locked memory       (kbytes, -l) 64
max memory size         (kbytes, -m) unlimited
open files                      (-n) 1024
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 8192
cpu time               (seconds, -t) unlimited
max user processes              (-u) unlimited
virtual memory          (kbytes, -v) unlimited
file locks                      (-x) unlimited

UPD: Solution: Disable journal.

Answer 1

The error you are getting suggests that the data is still there, but something is causing a problem when it mmaps the data files. This line is the first thing you need to look at:

Thu Jul 26 12:05:44 [conn5] Assertion: 13636:file /var/lib/mongodb/baza.4 open/create failed in createPrivateMap (look in log for more information)

Check the permissions on that file (and other files in the data path), make sure nothing is wrong from that perspective.

Next, there is the out of memory error:

Thu Jul 26 12:05:44 [conn5] ERROR: mmap private failed with out of memory. (64 bit build)

You might be running into this bug: https://jira.mongodb.org/browse/SERVER-6414

So, you can try 2.0.7 which contains a fix for this, the release candidate is available on the downloads page as of writing this.

Finally, if you have the option you may be able to solve this by increasing memory on this instance (if it is a virtual machine of some sort) 1.4GB is not a lot of RAM in terms of running a MongoDB instance.