writeFileSync only write half of data

Question

So i encountered a problem while doing my project.The problem is that when i try to write my data to csv file,it only write half of the data ,even sometimes only less than half of my data.I don't know what the problem is because there is no error shown in the terminal.

Below is my code

async function getFile(req, res, next) {

    try {
        let URI;
        const listOfKeys = await listAllKeys();
        let temp = []

        await Promise.all(listOfKeys.map(async function (data) {
            let response = await writeFile(data.Key);
            temp.push(response)
        }))
            .then(async _ => {
                fs.writeFileSync(FILE_PATH, Buffer.concat(temp));
            })
            .catch(err => {
                console.log(err)
            })
        return res.json({ message: 'halo' });
    } catch (err) {
        console.log('hmm.... error', err);
        return next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
    };
};

And this is the writeFile function

function writeFile(key) {
    return new Promise((resolve, reject) => {
        s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
            if (err) reject(err)
            else resolve(data.Body)
        })
    });
};

If possible, i would like to know the detail of my problem and how to fix it.Thanks.

Answer 1

It looks to me like you can do it like this (function names have been modified to make sense to me):

const fsp = require('fs').promises;

async function getDataAndWriteFile(req, res, next) {
    try {
        let URI;
        const listOfKeys = await listAllKeys();

        let responses = await Promise.all(listOfKeys.map(function (data) {
            return getData(data.Key);
        }));
        await fsp.writeFile(FILE_PATH, Buffer.concat(responses);
        res.json({ message: 'halo' });
    } catch(err) {
        console.log('hmm.... error', err);
        next(new APIError(err, httpStatus.INTERNAL_SERVER_ERROR, true));
    }
}

function getData(key) {
    return new Promise((resolve, reject) => {
        s3.getObject({ Bucket: process.env.AWS_BUCKET, Key: key }, (err, data) => {
            if (err) reject(err)
            else resolve(data.Body)
        })
    });
}

Summary of changes:

1. Change function names to better describe what they do
2. Use let responses = await Promise.all() to get the data from the promise array.
3. Use the promise interface in the fs module with await fsp.writeFile() to write the data out to your file.
4. Use try/catch to catch all the promise rejections in one place

Possible Open Issues:

1. Writing this Buffer.concat(responses) to disk seems kind of odd. Is that really what you want in this file?