High RAM usage in simple node.js app

Question

I made this simple node.js app:

var express = require('express');
var app = express();
var alasql = require('alasql');
var url = require('url');

var port = 3000;

app.get('/getuser', function(request, response) {
    var queryObject = url.parse(request.url, true).query;
    var country = queryObject.country;
    var nameset = queryObject.nameset;
    var state = queryObject.state;
    var selectAll;

    if (country == "US" && state !== null && state !== "") {
        alasql.promise("SELECT * from JSON('database.json') WHERE Country = ? AND NameSet = ? AND State = ?", [country, nameset, state]).
        then(function(res) {
            if (res !== null && res.length > 0) {
                var result = res[Math.floor(Math.random() * res.length + 1)];
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end(JSON.stringify(result));
            } else {
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end("noresults");
            }
        }).catch(function(err) {
            console.log('Does the database.json file exists? there was an error:', err);
        });


    } else {
        alasql.promise("SELECT * from JSON('database.json') WHERE Country = ? AND NameSet = ?", [country, nameset]).
        then(function(res) {
            if (res !== null && res.length > 0) {
                var result = res[Math.floor(Math.random() * res.length + 1)];
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end(JSON.stringify(result));
            } else {
                response.writeHead(200, {
                    'Content-Type': 'text/plain'
                });

                response.end("noresults");
            }
        }).catch(function(err) {
            console.log('Does the database.json file exists? there was an error:', err);
        });
    }
});

app.listen(port);

As you can see, I load database.json file, do some alasql query on it and return that data as response. database.json is ~50MB in size. My problem is that this app uses 400-600MB of RAM!

I made this version too, where I open database.json file using .require, but memory consumption is same!

var express = require('express');
var app = express();
var alasql = require('alasql');
var url = require('url');

var port = 3000;

var database = require('./database.json');


app.get('/getuser', function(request, response) {
    var queryObject = url.parse(request.url, true).query;
    var country = queryObject.country;
    var nameset = queryObject.nameset;
    var state = queryObject.state;
    var selectAll;

    if (country == "US" && state !== null && state !== "") {
        selectAll = alasql("SELECT * from ? WHERE Country = ? AND NameSet = ? AND State = ?", [database, country, nameset, state]);
    } else {
        selectAll = alasql("SELECT * from ? WHERE Country = ? AND NameSet = ?", [database, country, nameset]);
    }

    if (selectAll !== null && selectAll.length > 0) {
        var res = selectAll[Math.floor(Math.random() * selectAll.length + 1)];

        response.writeHead(200, {
            'Content-Type': 'text/plain'
        });

        response.end(JSON.stringify(res));

    } else {
        response.writeHead(200, {
            'Content-Type': 'text/plain'
        });

        response.end("noresults");
    }
});

app.listen(port);

What am I doing wrong, I suppose it is some issue regarding the size of my JSON file? It is array of 55000 records, each one having 20 fields, if that info helps you?

Answer 1

The fact that it's 50mb on disk, isn't particularly useful when comparing into memory.

Depending on what's in your database JSON, it could be significantly larger than the on disk representation.

Take an integer for example, if you have the value 5 in your JSON it only takes up 1 byte in ASCII, but it will take up 8 bytes in Javascript as all numbers are 8 bytes. When you take into account the number of references required to keep track of the JSON object itself and it's children, etc, you can end up with something much larger than the on-disk representation.