Mongoose duplicate key error with upsert

Question

I have problem with duplicate key. Long time can`t find answer. Please help me solve this problem or explain why i get duplicate key error.

Trace: { [MongoError: E11000 duplicate key error collection: project.monitor index: _id_ dup key: { : 24392490 }]
name: 'MongoError',
message: 'E11000 duplicate key error collection: project.monitor index: _id_ dup key: { : 24392490 }',
driver: true,
index: 0,
code: 11000,
errmsg: 'E11000 duplicate key error collection: project.monitor index: _id_ dup key: { : 24392490 }' }
at /home/project/app/lib/monitor.js:67:12
at callback (/home/project/app/node_modules/mongoose/lib/query.js:2029:9)
at Immediate.<anonymous> (/home/project/app/node_modules/kareem/index.js:160:11)
at Immediate._onImmediate (/home/project/app/node_modules/mquery/lib/utils.js:137:16)
at processImmediate [as _immediateCallback] (timers.js:368:17)

but in monitor i use upsert, so why i get duplicate error??

monitor.js:62-70

monitor schema

var monitorSchema = db.Schema({
   _id      : {type: Number, default: utils.minute},
   maxTicks : {type: Number, default: 0},
   ticks    : {type: Number, default: 0},
   memory   : {type: Number, default: 0},
   cpu      : {type: Number, default: 0},
   reboot   : {type: Number, default: 0},
streams  : db.Schema.Types.Mixed
}, {
collection: 'monitor',
strict: false
});

index

monitorSchema.index({_id: -1});
Monitor = db.model('Monitor', monitorSchema);

and increase by property

exports.increase = function (property, incr) {
    var update = {};
    update[property] = utils.parseRound(incr) || 1;
    Monitor.update({_id: utils.minute()}, {$inc: update}, {upsert: true}, function (err) {
        if (err) {
            console.trace(err);
        }
    });
};

utils.js

exports.minute = function () {
    return Math.round(Date.now() / 60000);
};

exports.parseRound = function (num, round) {
    if (isNaN(num)) return 0;
    return Number(parseFloat(Number(num)).toFixed(round));
};

Answer 1

An upsert that results in a document insert is not a fully atomic operation. Think of the upsert as performing the following discrete steps:

1. Query for the identified document to upsert.
2. If the document exists, atomically update the existing document.
3. Else (the document doesn't exist), atomically insert a new document that incorporates the query fields and the update.

So steps 2 and 3 are each atomic, but another upsert could occur after step 1 so your code needs to check for the duplicate key error and then retry the upsert if that occurs. At that point you know the document with that _id exists so it will always succeed.

For example:

var minute = utils.minute();
Monitor.update({ _id: minute }, { $inc: update }, { upsert: true }, function(err) {
    if (err) {
        if (err.code === 11000) {
            // Another upsert occurred during the upsert, try again. You could omit the
            // upsert option here if you don't ever delete docs while this is running.
            Monitor.update({ _id: minute }, { $inc: update }, { upsert: true },
                function(err) {
                    if (err) {
                        console.trace(err);
                    }
                });
        }
        else {
            console.trace(err);
        }
    }
});

See here for the related documentation.

You may still be wondering why this can happen if the insert is atomic, but what that means is that no updates will occur on the inserted document until it is completely written, not that no other insert of a doc with the same _id can occur.

Also, you don't need to manually create an index on _id as all MongoDB collections have a unique index on _id regardless. So you can remove this line:

monitorSchema.index({_id: -1}); // Not needed