Auto-increment field in MongoDb with async bulk insertion

Question

I'm looking at this article about auto-incrementing fields in MongoDb. http://docs.mongodb.org/manual/tutorial/create-an-auto-incrementing-field/

This seems good in theory, but doesn't work in my application where all DB calls are async. Ps, Im using Mongoskin. Consider this:

var getNextSequence function (name) {
  db.counters.findAndModify({_id: 'invoiceNr'}, {}, {$inc: {seq: 1}}, {new: true}, function(err, doc) {
    if (err) {
      return next(err);
    }
    return doc.seq;
  });
}; 

var seq = getNextSequence('invoiceNr');
console.log(seq);

This (Console.log) will of course not have any value when executed...

This makes me thinking about nested callbacks instead (classic Node style):

db.counters.findAndModify({_id: 'invoiceNr'}, {}, {$inc: {seq: 1}}, {new: true}, function(err, doc) {
  if (err) {
    return next(err);
  }
  var seq = getNextSequence('invoiceNr');
  // Do something with the seq, like using it when inserting another document.
});

This will work fine for one document, but I cant see how this would work if I make a bulk Insert (passing an array of several documents to insert). And I NEED bulk insertions.

Going back to loops and single insertions doesnt seems like a good solutions.

Do you have any good solution for this?

Answer 1

So the documentation recommends .findAndModify() for this type of operation because the actions of "updating/incrementing" and retrieving the result are an "atomic" operation. Of course if you are inserting 500 documents at a time you don't want to go back an "get the next increment" 500 times, because that would be insane. But there is something in the concept you are missing.

Who said you have to increment by 1? Simply put, when you know you are inserting 500 items, then you just want to increment the current counter by 501 so that the next operation that asks for the next counter value gets the value that has already been increased by 501, so that part is okay.

Now that you asked the counter to be incremented by a block of 500 you basically just use them. So you know that the first value to use basically starts at your return value minus 500, and you want to end your insert on the value that was actually returned.

So you want some code like this:

var async = require('async'),
    mongo = require('mongoskin'),
    db = mongo.db('mongodb://localhost/test');


// The general incrementer

function getNextSequence( name, increment, callback ) {

  // Handling passing in "increment" as optional
  var args = Array.prototype.slice.call(arguments, 1);

  callback = args.pop();
  increment = args.length ? args.shift() || 1 : 1; // default 1

  db.collection('counters').findAndModify(
    { "_id": name },
    {},
    { "$inc": { "seq": increment } },
    { "new": true, "upsert": true },
    function(err,doc) {
      callback(err,doc.seq);
    }
  );

}

// Meat of your insertion logic

db.collection('target',function(err, collection) {

  var arrayToInsert = []; // simulating, but 500 items long assumed
  for (var x=0; x < 500; x++) {
    arrayToInsert.push({
      "x": x
    });
  }

  var bulk = collection.initializeOrderedBulkOp();

  async.waterfall(
    [
      function(callback) {
        getNextSequence('invoiceNo',arrayToInsert.length + 1,callback);
      },
      function(seq,callback) {
        var current = seq - arrayToInsert.length;
        var index = 0;

        async.whilst(
          function() { return index < arrayToInsert.length },
          function(callback) {
            doc = arrayToInsert[index];
            doc["invoice"] = current;
            bulk.insert( doc );
            index++;
            current++
            callback();
          },
          function(err) {
            bulk.execute(callback);
          }
        );
      }
    ],
    function(err,result) {
      if (err) throw err;
      // Log BulkWrite result
      console.log( JSON.stringify( result, undefined, 2 ) );
    }
  );

});

Well, that's the basic simulator but you should get the idea. Essentially just grab a block of "values" for how many entries you intend to insert and use each of the "values" as you insert the actual documents.

Worst case scenario? The inserts failed somewhere and not all of the values ended up in documents. So what? It does not matter, as the point is you already incremented the counter and no other process can possibly grab those same "values" from the counter.

As you can gather from the comments, this is where I was trying to lead you and you may have worked out the basics already. Nothing says you can't increment that counter by more than 1, so just increment by the amount you want to.