We are writing a script that reads a large set of JPG files on our server (infinite, since we have another process that keeps writing JPG files to the same directory) and send them to users' browsers as an MJPEG stream at fixed time interval (variable "frameDelay" in code below). This is similar to what an IP camera would do.
We found out the memory usage of this script keeps going up and always ends up being killed by the system (Ubuntu);
We have inspected this seemingly simple script many many times. Therefore I'm posting the code below. Any comments / suggestions are greatly appreciated!
app.get('/stream', function (req, res) {
res.writeHead(200, {
'Content-Type':'multipart/x-mixed-replace;boundary="' + boundary + '"',
'Transfer-Encoding':'none',
'Connection':'keep-alive',
'Expires':'Fri, 01 Jan 1990 00:00:00 GMT',
'Cache-Control':'no-cache, no-store, max-age=0, must-revalidate',
'Pragma':'no-cache'
});
res.write(CRLF + "--" + boundary + CRLF);
setInterval(function () {
if(fileList.length<=1){
fileList = fs.readdirSync(location).sort();
}else{
var fname = fileList.shift();
if(fs.existsSync(location+fname)){
var data = fs.readFileSync(location+fname);
res.write('Content-Type:image/jpeg' + CRLF + 'Content-Length: ' + data.length + CRLF + CRLF);
res.write(data);
res.write(CRLF + '--' + boundary + CRLF);
fs.unlinkSync(location+fname);
}else{
console.log("File doesn't find")
}
}
console.log("new response:" + fname);
}, frameDelay);
});
app.listen(port);
console.log("Server running at port " + port);
To facilitate the troubleshooting process, below is a stand-alone (no 3rd-party lib) test case.
It has exactly the same memory issue (memory usage keeps going up and finally got killed by the OS).
We believe the problem is in the setInterval () loop - maybe those images didn't get deleted from memory after being sent or something (maybe still stored in variable "res"?).
Any feedback / suggestions are greatly appreciated!
var http = require('http');
var fs = require('fs');
var framedelay = 40;
var port = 3200;
var boundary = 'myboundary';
var CR = '\r';
var LF = '\n';
var CRLF = CR + LF;
function writeHttpHeader(res)
{
res.writeHead(200,
{
'Content-Type': 'multipart/x-mixed-replace;boundary="' + boundary + '"',
'Transfer-Encoding': 'none',
'Connection': 'keep-alive',
'Expires': 'Fri, 01 Jan 1990 00:00:00 GMT',
'Cache-Control': 'no-cache, no-store, max-age=0, must-revalidate',
'Pragma': 'no-cache',
});
res.write(CRLF + '--' + boundary + CRLF);
}
function writeJpegFrame(res, filename)
{
fs.readFile('./videos-8081/frames/' + filename, function(err, data)
{
if (err)
{
console.log(err);
}
else
{
res.write('Content-Type:image/jpeg' + CRLF);
res.write('Content-Length:' + data.length + CRLF + CRLF);
res.write(data);
res.write(CRLF + '--' + boundary + CRLF);
console.log('Sent ' + filename);
}
});
}
http.createServer(function(req, res)
{
writeHttpHeader(res)
fs.readdir('./videos-8081/frames', function(err, files)
{
var i = -1;
var sorted_files = files.sort();
setInterval(function()
{
if (++i >= sorted_files.length)
{
i = 0;
}
writeJpegFrame(res, sorted_files[i]);
}, framedelay);
});
}).listen(port);
console.log('Server running at port ' + port);
There are a few things caused this
res.write().setInterval() is OK to use, and will not cause any problem as long as the client keep the connection. But when the client is disconnected, you need to stop it. To do so, you need to monitor 'close' event.setTimeout().Here is the fixed code.
var http = require('http');
var fs = require('fs');
var framedelay = 40;
var port = 3200;
var boundary = 'myboundary';
var CR = '\r';
var LF = '\n';
var CRLF = CR + LF;
http.createServer(function(req, res) {
var files = fs.readdirSync('./imgs');
var i = -1;
var timer;
var sorted_files = files.sort();
res.writeHead(200, {
'Content-Type': 'multipart/x-mixed-replace;boundary="' + boundary + '"',
'Transfer-Encoding': 'none',
'Connection': 'keep-alive',
'Expires': 'Fri, 01 Jan 1990 00:00:00 GMT',
'Cache-Control': 'no-cache, no-store, max-age=0, must-revalidate',
'Pragma': 'no-cache',
});
res.write(CRLF + '--' + boundary + CRLF);
var writePic = function() {
if (++i >= sorted_files.length)
i = 0;
var data = fs.readFileSync('./imgs/' + sorted_files[i]);
res.write('Content-Type:image/jpeg' + CRLF);
res.write('Content-Length:' + data.length + CRLF + CRLF);
res.write(data);
var ok = res.write(CRLF + '--' + boundary + CRLF);
console.log('Sent ' + sorted_files[i], ok);
if (ok)
timer = setTimeout(writePic, framedelay);
};
res.on('close', function() {
console.log('client closed');
clearTimeout(timer);
});
res.on('drain', function() {
console.log('drain');
timer = setTimeout(writePic, framedelay);
});
}).listen(port);
console.log('Server running at port ' + port);