Improving performance of inserting into Mongo from ActiveMQ Artemis

Question

The basic idea of the following code is I read messages off an ActiveMQ Artemis installation and insert them into a MongoDB instance.

It works well for up to a hundred or so messages per second but crashes if I throw a few thousand at it. My first guess would be the constant opening and closing of database connections. Should I also think about using an in-memory store and doing bulk database inserts?

The code is all running in node using the mqtt and mongodb npm packages. The code below, the database and the queue are all running in docker containers if it makes any difference.

var mqtt = require('mqtt'),
client = mqtt.connect('mqtt://mq:1883', {
    username: "*************",
    password: "*************"
}),
MongoClient = require('mongodb').MongoClient,
ObjectId = require('mongodb').ObjectID,
assert = require('assert'),
url = 'mongodb://db:27017/uo-readings';

client.on('connect', function () {
    client.subscribe('readings');
});

client.on('error', function(error){
    console.log(error)
});

client.on('message', function (topic, message) {

    console.log(message.toString());

    MongoClient.connect(url, function(err, db) {
        assert.equal(null, err);

        console.log("Connected correctly to server.");

        db.collection('readings').insertOne(JSON.parse(message.toString()), function(err, result) {
            assert.equal(err, null);
            console.log("Inserted a document into the readings collection.");
        });

        client.end(function(){
            console.log("Closing Connection.");
            db.close();
        });
    });
});

Answer 1

See @Jonathan Muller's comment above