javascriptnode.jsoracle-databasenode-oracledb
NJS-024: memory allocation failed in OracleDB - Nodejs
I am trying to run a query using OracleDB with Nodejs to get the view populated in the UI but I get a NJS-024: memory allocation failed error. Can someone help me out? The view contains 120 columns in total and when I query the view in SQL Developer, it works just fine.
ConnectionPool.js:
var path = require('path');
var oracledb = require('oracledb');
var poolMap = {};
var logger = require(path.join(global.root + '/app/util/logger.js'))();
function createPool(poolName, config, callback) {
oracledb.createPool(
config,
function(err, p) {
if (err){
logger.error(err);
return;
}
poolMap[poolName] = p;
callback(poolMap[poolName]);
}
);
}
function getPool(poolName) {
return poolMap[poolName];
}
module.exports = {
createPool: createPool,
getPool: getPool
};
This is my poolAttributes:
var pool;
oracledb.prefetchRows = 10000;
oracledb.maxRows = 400000;
var poolAttrs = {
user: dbcfg.username,
password: dbcfg.password,
connectString: dbcfg.connectionString,
connectionClass : 'Report API',
poolMin : 3,
poolMax : 10,
poolIncrement: 2,
poolTimeout : 600 //seconds
};
connectionPool.createPool("Reports", poolAttrs, function(connPool){
pool = connPool;
logger.info("Pool created by reports.");
});
This is my code:
router.post('/report/', jsonParser, function (req, res) {
var data = req.body,
startRow = data.startRow,
numRows = data.numRows,
sortCol = data.sortCol,
sortDir = data.sortDir;
var countQuery = 'SELECT COUNT(*) ' +
'FROM this_view ' ;
var query = 'SELECT * ' +
'FROM this_view' ;
var seg,
orderBy,
offset;
orderBy = ' ORDER BY UPPER(' + sortCol + ') ' + sortDir;
offset = ' OFFSET ' + startRow + ' ROWS FETCH NEXT ' + numRows + ' ROWS ONLY';
query += orderBy;
query += offset;
logger.info("Begin: " + (new Date().toString()));
async.parallel({
rows: function (callback) {
pool.getConnection(function (err, connection) {
logger.info("Begin Connection: " + (new Date().toString()));
if (err) {
logger.error(err.message);
return;
}
logger.info("Begin execute: " + (new Date().toString()));
connection.execute(
query,
{},
{
resultSet: true,
prefetchRows: 1000
},
function (err, results) {
logger.info("End execute: " + (new Date().toString()));
var rowsProcessed = 0;
var startTime;
if (err) {
logger.error(err.message);
callback("Something broke in the first thing");
doRelease(connection);
return;
}
var procJson = [];
function fetchRowsFromRS(connection, resultSet, numRows) {
resultSet.getRows(
numRows, // get this many rows
function (err, rows) {
if (err) {
console.error(err);
doClose(connection, resultSet); // always close the result set
} else if (rows.length >= 0) {
/**
* For each row in the result, pushes a new object to the rows array
* In each new object, the key is assigned and the result row value set
*/
for (var i = 0; i < rows.length; i++) {
procJson.push({});
console.log(procJson);
for (var j = 0; j < resultSet.metaData.length; j++) {
procJson[i][resultSet.metaData[j].name.toLowerCase()] = rows[i][j];
}
}
//TODO: Add null handling
logger.info("Send JSON: " + (new Date().toString()));
logger.info("JSON Sent: " + (new Date().toString()));
if (rows.length === numRows) // might be more rows
fetchRowsFromRS(connection, resultSet, numRows);
else
doClose(connection, resultSet); // always close the result set
} else { // no rows
doClose(connection, resultSet); // always close the result set
}
});
}
fetchRowsFromRS(connection, result.resultSet, numRows);
callback(null, procJson);
});
});
},
totalRows: function (callback) {
pool.getConnection(function (err, connection) {
logger.info("Begin Connection: " + (new Date().toString()));
if (err) {
logger.error(err.message);
return;
}
logger.info("Begin execute: " + (new Date().toString()));
connection.execute(
countQuery,
function (err, result) {
logger.info("End execute: " + (new Date().toString()));
if (err) {
logger.error(err.message);
callback("Something broke");
doRelease(connection);
return;
}
logger.info("Send JSON: " + (new Date().toString()));
console.log(result.rows);
callback(null, result.rows[0][0]);
logger.info("JSON Sent: " + (new Date().toString()));
doRelease(connection);
});
});
}
}, function(err, result){
if(err){
logger.error(err);
}
res.send(result);
});
});
If rows.length >=0 and if the query returns 0 results, I get this.
Solution
How much memory does your Node.js server have? You're setting maxRows very high and grabbing all the data in a single shot. This is likely causing you to run out of memory. Generally, the key is to balance round trips (which you want to reduce) with memory usage (which goes up as round trips are reduced.
You'll want to leverage the ResultSet API, which allows you to stream a read-consistent view of data in smaller chunks. Have a look at this for ideas: https://jsao.io/2015/07/an-overview-of-result-sets-in-the-nodejs-driver/
Rather than buffer the data in the Node.js server (which would lead to the same problem), you'll want to stream it down to the http request.
Finally, but perhaps most importantly, note that your code is currently open to SQL injection. Values that come in from users via req.body cannot be trusted. They must either be bound in using bind variables OR sanitized using something like dbms_assert.
Only values (like numRows) can be bound in. Identifiers (like sortCol) have to be sanitized. You'll likely want to do the sanitization in Node.js, so here's a really basic check that should help.
You could create an "assert" module:
function simpleSqlName(name) {
if (name.length > 30) {
throw new Error('Not simple SQL');
}
// Fairly generic, but effective. Would need to be adjusted to accommodate quoted identifiers,
// schemas, etc.
if (!/^[a-zA-Z0-9#_$]+$/.test(name)) {
throw new Error('Not simple SQL');
}
return name;
}
module.exports.simpleSqlName = simpleSqlName;
function validSortOrder(order) {
if (order !== 'desc' && order !== 'asc') {
throw new Error('Not valid sort order');
}
return order;
}
module.exports.validSortOrder = validSortOrder;
Then your code would look more like this (notice I'm using both the assert module and bind variables):
let assert = require('assert.js');
router.post('/report/', jsonParser, function (req, res) {
var data = req.body,
startRow = data.startRow,
numRows = data.numRows,
sortCol = assert.simpleSqlName(data.sortCol),
sortDir = assert.validSortOrder(data.sortDir);
var countQuery = 'SELECT COUNT(*) ' +
'FROM this_view ' ;
var query = 'SELECT * ' +
'FROM this_view' ;
var seg,
orderBy,
offset;
orderBy = ' ORDER BY UPPER(' + sortCol + ') ' + sortDir;
offset = ' OFFSET :start_row ROWS FETCH NEXT :num_rows ROWS ONLY';
query += orderBy;
query += offset;
logger.info("Begin: " + (new Date().toString()));
async.parallel({
rows: function (callback) {
pool.getConnection(function (err, connection) {
logger.info("Begin Connection: " + (new Date().toString()));
if (err) {
logger.error(err.message);
return;
}
logger.info("Begin execute: " + (new Date().toString()));
connection.execute(
query,
{
start_row: startRow,
num_rows: numRows
},
function (err, result) {
logger.info("End execute: " + (new Date().toString()));
if (err) {
logger.error(err.message);
callback("Something broke in the first thing");
doRelease(connection);
return;
}
console.log(result.rows);
var procJson = [];
/**
* For each row in the result, pushes a new object to the rows array
* In each new object, the key is assigned and the result row value set
*/
for (var i = 0; i < result.rows.length; i++) {
procJson.push({});
for (var j = 0; j < result.metaData.length; j++) {
procJson[i][result.metaData[j].name.toLowerCase()] = result.rows[i][j];
}
}
logger.info("Send JSON: " + (new Date().toString()));
callback(null, procJson);
logger.info("JSON Sent: " + (new Date().toString()));
doRelease(connection);
});
});
},
totalRows: function (callback) {
pool.getConnection(function (err, connection) {
logger.info("Begin Connection: " + (new Date().toString()));
if (err) {
logger.error(err.message);
return;
}
logger.info("Begin execute: " + (new Date().toString()));
connection.execute(
countQuery,
function (err, result) {
logger.info("End execute: " + (new Date().toString()));
if (err) {
logger.error(err.message);
callback("Something broke");
doRelease(connection);
return;
}
logger.info("Send JSON: " + (new Date().toString()));
console.log(result.rows);
callback(null, result.rows[0][0]);
logger.info("JSON Sent: " + (new Date().toString()));
doRelease(connection);
});
});
}
}, function(err, result){
if(err){
logger.error(err);
}
res.send(result);
});
});
Learn more about bind variables here: https://github.com/oracle/node-oracledb/blob/master/doc/api.md#bind
Also, check out the slides from a recent talk I gave. You may get something out of them... https://www.dropbox.com/s/2rhnu74z2y21gsy/Tips%20and%20Tricks%20for%20Getting%20Started%20with%20the%20Oracle%20Database%20Driver%20for%20Node.pdf?dl=0
- How to fetch data server-side in the latest Next.js? Tried getStaticProps but it's not running and getting undefined
- How can I split a string containing emoji into an array?
- nodemailer Invalid login: 535 Authentication Failed
- Jquery Remove All Event Handlers Inside Element
- Multiple Vimeo video in multiple modals
- How to convert ReadStream into ReadableStream in NodeJS?
- How to create a backdrop for a dialog?
- How can an object have a `.constructor` property if I reset its prototype to an empty object (`{}`)?
- Different way of showing and hiding certain elements
- Is there a good use case for the constructor property in Javascript?
- How to make search in select box?
- My scrollView compnent is not working despite all the online solutions
- Why does my linter warn “Use object destructuring (prefer destructure)”?
- Is it possible to keep scheduled tasks from overlapping in NestJS?
- export html table as word file and change file orientation
- Vis.js callback when the network finishes loading
- Is it safe to use the `Function` constructor to validate JavaScript syntax?
- Equivalent of destructors in JavaScript's object model
- How to slow down scroll to top speed?
- How can I tell if it is an object or an array?
- Bottom of gradient text gets cut off tailwindcss
- How to share state between reducers in redux (how to organize redux)?
- I'm trying to create a password generator
- how to convert a raw readme file to html formated document after geting it from github in angular js
- How to use npm Marked with HighlightJS
- Avoid browser popup blockers
- What does this function do? And what is the odd syntax?
- Next.js server side component request, how to send my cookie token?
- Javascript single quote inside string for exported text file
- Passing in a result of a function to generate readme