I have a simple webserver created in Express.js. The server serves files that are dynamically created by processing data from a third-party API.
Here is my webserver code, it requests builder.js to build the file, which requests, receives, processes and returns data from a third-party API by awaiting the response of a promisified oauth request as needed. The builder will at least call the API two or more times to create a complete file ready for serving.
const express = require('express');
const app = express();
const builder = require('./builder.js');
let requestID = 0;
app.get('/', function (req, res) {
requestID++;
console.log(`tab: ${requestID}<`);
res.set('Content-Type', 'text/plain; charset=UTF-8')
res.status(200).send('Hello World');
console.log(`tab: ${requestID}>`);
});
app.get('/file', async function (req, res) {
requestID++;
console.log(`tab: ${requestID}<`);
if (req.query["id"] != undefined) {
let URLparams = new URLSearchParams(req.query);
console.log(`tab: ${requestID}, requested id: ${URLparams.get("id")}`);
let output = await builder.buildFile(URLparams);
try {
console.log(`tab: ${requestID}, requested id: ${URLparams.get("q")}, served ${getIDfromOutput(output)}`);
res.set('Content-Type', 'application/rss+xml; charset=UTF-8')
res.status(200).send(output);
} catch(e) {
console.log(`tab: ${requestID}, ${e}`);
if (e instanceof String) { res.send(JSON.stringify(JSON.parse(e), null, 3)); }
else { res.send(JSON.stringify(e, null, 3)); }
};
} else {
res.set('Content-Type', 'text/plain; charset=UTF-8')
res.status(404)
.send("404: Page not found.");
}
console.log(`tab: ${requestID}>`);
});
app.listen(3000, "localhost");
The code works as intended when making requests to the /file one at a time.
//1 tab loaded
tab: 1<
tab: 1, requested: 1331444331778101248
tab: 1, requested: 1331444331778101248, loaded 1331444331778101248
tab: 1>
However, when the endpoint is requested for multiple unique requests at the same time (opening multiple tabs at the same time or running parallel wget commands), the server either responds correctly in some cases, but it mostly responds with the same file served previously.
// 5 unique tabs loaded at the same time: 1551641441679597569, 1448115610173558787, 1370689539505860613, 1328121208022446086, 1509637745140019212
tab: 1<
tab: 1, requested: 1551641441679597569
tab: 2<
tab: 2, requested: 1448115610173558787
tab: 2, requested: 1551641441679597569, loaded 1551641441679597569
tab: 2>
tab: 3<
tab: 3, requested: 1370689539505860613
tab: 3, requested: 1448115610173558787, loaded 1448115610173558787
tab: 3>
tab: 3, requested: 1370689539505860613, loaded 1370689539505860613
tab: 3>
The result of these simultaneous requests causes tabs 1-4 load fine, but tab 5 shows the output of tab 4. The console logger can't also seem to show the issue, but it's definitely different to the normal, one-off request.
I do not want this to happen as I fear that this may happen in production, and I do not want the outputs to be leaked across requests. However, I have no idea what is causing this or how to investigate to fix this issue. The code works fine when builder.buildFile() has to make one API call to the third-party, but I am always making 2 or more calls.
As requested, here is the stripped version of the buildFile function. When serving directly from the API through builder.getData(url), the responses for each request is unique and the responses do not cross over other requests. It happens only when it goes through builer.buildFile(). I feel like the issue is with the way the promises are handles, but I am not sure.
const OAuth = require('oauth');
const { promisify } = require('util');
require('dotenv').config();
let oauth = getOAuth();
module.exports = {
buildFile: async function (urlParam) { //URLSearchParams(req.query)
let id = urlParam.get("id");
try {
let metadata = await getData(`http://API/id=${id}?type=metadata`);
let XMLFile = await fileBuilder(metadata);
return XMLFile;
} catch (e) {
console.log("Err: ");
console.log(e);
return Promise.reject(e);
}
},
getData: async function (url) {
return await getData(url);
}
}
async function fileBuilder(metadata) {
let response = await getData(`$http://API/id=${id}?type=fulldata`);
response = extendData(response); //calls await getData() once more to fill in any gaps in the initial response
let xml = ``;
/* Build XMl file as a string, appending as the function processes the data*/
return xml;
}
function getOAuth() {
return new OAuth.OAuth(
'https://API/request_token',
'https://API/access_token',
process.env.api_key,
process.env.api_secret,
'1.0A', null, 'HMAC-SHA1'
);
}
async function getData(url) {
const get = promisify(oauth.get.bind(oauth));
let body;
try {
body = await get(
url,
process.env.access_key,
process.env.access_secret
);
} catch (e) {
console.log("getData failed: \n" + JSON.stringify(e));
return Promise.reject(e);
}
return JSON.parse(body);
}
You see the mixup in the console because you are using the shared requestId after it was changed. In order to avoid this you need to fix it at the beginning of the function.
The problem you have with the wrong file being served might come from the buildFile function since I can't locate it in this code fragment.
app.get('/file', async (req, res) => {
const localReqId = requestID++;
console.log(`tab: ${localReqId}<`);
if (req.query.id !== undefined) {
const URLparams = new URLSearchParams(req.query);
console.log(`tab: ${localReqId}, requested id: ${URLparams.get('id')}`);
const output = await builder.buildFile(URLparams);
try {
console.log(`tab: ${localReqId}, requested id: ${URLparams.get('q')}, served ${getIDfromOutput(output)}`);
res.set('Content-Type', 'application/rss+xml; charset=UTF-8');
res.status(200).send(output);
} catch (e) {
console.log(`tab: ${localReqId}, ${e}`);
if (e instanceof String) {
res.send(JSON.stringify(JSON.parse(e), null, 3));
} else {
res.send(JSON.stringify(e, null, 3));
}
}
} else {
res.set('Content-Type', 'text/plain; charset=UTF-8');
res.status(404)
.send('404: Page not found.');
}
console.log(`tab: ${localReqId}>`);
});