Maximum stack size exceeded - event-stream

Question

Having this piece of code:

function query(url, dbName, collection, filter, requestId) {
    MongoClient.connect(url, {native_parser:true, authSource:'admin'}, function(err, client) {
        if (err) {
            throw err;
        }
        const db = client.db(dbName);
        var stream = db.collection(collection).find(filter, {fields:{_id: 0}}).stream();

        var fileName = '/opt/' + requestId + '.txt';
        var writer = fs.createWriteStream(fileName);
        writer.write('[\n');

        stream.on('end', function(){
            writer.write('\n]');
        });

        stream.pipe(es.map(function (doc, next) {
            doc = JSON.stringify(doc);
            next(null, doc);
        })).pipe(es.join(',\n')).pipe(writer).on('close', function(){
            sftp.put(fileName, '/opt/' + requestId + '.txt')
                .then(logger.info('Done uploading the file via SFTP'));

            mqttClient.publish('response', 'The CSV for requestId has been uploaded FTP');
        });
    });
}

The problem is that when the query returns large number of documents the function will fail with

/node_modules/map-stream/index.js:103
        throw err
        ^

RangeError: Maximum call stack size exceeded
    at Stream.ondata (internal/streams/legacy.js:14:18)
    at emitOne (events.js:116:13)
    at Stream.emit (events.js:211:7)
    at Stream.<anonymous> (/node_modules/event-stream/index.js:298:12)
    at Stream.stream.write (/node_modules/through/index.js:26:11)
    at Stream.ondata (internal/streams/legacy.js:16:26)
    at emitOne (events.js:116:13)
    at Stream.emit (events.js:211:7)
    at queueData (/node_modules/map-stream/index.js:43:21)
    at next (/node_modules/map-stream/index.js:71:7)
    at /node_modules/map-stream/index.js:85:7
    at /opt/subscriber.js:84:7
    at wrappedMapper (/node_modules/map-stream/index.js:84:19)
    at Stream.stream.write (/node_modules/map-stream/index.js:96:21)
    at Cursor.ondata (_stream_readable.js:639:20)
    at emitOne (events.js:116:13)

What this functions is doing is getting a filter, running a mongodb query based on the filter and write the resulted documents into a file that is then ftp-ed.

The function fails at next(null, doc);

Any suggestions on how to improve the code for not having to increase the call stack size ?

Answer 1

well i never worked with this lib although it seems very popular . can you try work with the events of the steam to do the manipulation and see if it works ?

function query(url, dbName, collection, filter, requestId) {
    MongoClient.connect(url, {native_parser: true, authSource: 'admin'}, function (err, client) {
        if (err) {
            throw err;
        }
        const db = client.db(dbName);
        var stream = db.collection(collection).find(filter, {fields: {_id: 0}}).stream();

        var fileName = '/opt/' + requestId + '.txt';
        var writer = fs.createWriteStream(fileName);
        writer.write('[\n');

        stream.on('data', function (doc) {
            writer.write(`${JSON.stringify(doc)}\n`);
        });

        stream.on('end', function () {
            writer.write('\n]');
            sftp.put(fileName, '/opt/' + requestId + '.txt')
                .then(logger.info('Done uploading the file via SFTP'));

            mqttClient.publish('response', 'The CSV for requestId has been uploaded FTP');
        });
    });
}