Busboy finishes before parsing all data

Question

I have an express application and have a function that uses busboy to parse the form data which returns the field values of the form but do not parse the whole fields before the return call.

module.exports = async function (headers, invalidMime, res ,req) {
    let fields = {};
    const fileWrites = []
    let filesToUpload = [];
    let finished = false;
    const busboy = new Busboy({
        headers: headers,
        limits: { ileSize: 10 * 1024 * 1024 }
    });

    await busboy.on("field", (fieldname, val) => {
        console.log(fieldname); // Log 1
        fields[fieldname] = val;
    });

    await busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
        
        if (invalidMime(mimetype)) return res.status(404).json({ [fieldname]: "Wrong file type submitted" });
        
        file.on("limit", () => { return res.status(404).json({ general: "File size too large!" }); });

        const randomizedFileName = createFileName(filename);
        const filePath = path.join(os.tmpdir(), randomizedFileName);

        fileWrites.push(createImagePromise(file, filePath, randomizedFileName, mimetype, fieldname));
    });

    busboy.on("finish", () => {
      console.log(fields); // Log 2
    });
      
    busboy.end(req.rawBody);

    return {filesToUpload: fileWrites, fields: fields}
}

This return my fields but missing the last one. When I debug it with console log, I can see that busboy.on("finish") executes after I return my values which results in missing variables.

const formData = await  FormParser(req.headers, invalidMime, res, req);

console.log(formData); // Log 3
const { fields, filesToUpload } = formData;

  var1                     // Log 1 - Before return
  var2                     // Log 1 - Before return
  var3                     // log 1 - Before return
  { filesToUpload: [],     // Log 3 - After return
    fields:
     { var1: 'var1',
       var2: 'var2',
       var3: 'var3' } }          
  var4                     // Log 1 - After return
  { var1: 'var1',          // Log 2 - After return
    var2: 'var2',
    var3: 'var3',
    var4: 'var4' }                

How can I make busboy to return values when it is finished parsing?

Answer 1

Asychronous functions don't magically become synchronous when you sprinkle a few awaits here and there.

For example, this makes no sense:

await busboy.on("field", (fieldname, val) => {
    console.log(fieldname); // Log 1
    fields[fieldname] = val;
});

You think it somehow waits for the fields object to be filled, but what it really does is: It waits for the .on() function to return, and this function returns immediately. Its only job is to assign an event handler. The field event has not even occurred yet at all.

The solution in asynchronous programming is always: Do the work in the event handler that signifies that a task has completed. You're trying to do the work (return {filesToUpload: fileWrites, fields: fields}) in the last line of your function, as if the last line would be the last thing to run. That's not the case.

Once you move the bits that need to react to events inside the event handlers, you'll find that the whole function does not need to be async at all.

Disclaimer: the following code is untested, I have not used busboy before, do what it means, not necessarily what it says.

module.exports = function (headers, invalidMime, res, req) {
    let fields = {};
    let pendingFileWrites = [];

    const busboy = new Busboy({
        headers: headers,
        limits: { fileSize: 10 * 1024 * 1024 }
    });

    busboy.on("filesLimit", () => {
        res.status(400).json({ error: "File size too large!" });
    });
    busboy.on("error", () => {
        res.status(500).json({ error: "Error parsing data" });
    });
    busboy.on("field", (fieldname, val) => {
        fields[fieldname] = val;
    });
    busboy.on("finish", () => {
        Promise.all(pendingFileWrites).then((fileWrites) => {
            // NOW we're done
            res.json({
                filesToUpload: fileWrites, 
                fields: fields
            });
        });
    });

    busboy.on("file", (fieldname, file, filename, encoding, mimetype) => {
        console.log(`Processing [{filename}] ({mimetype})`);
        if (invalidMime(mimetype)) return res.status(404).json({ [fieldname]: "Wrong file type submitted" });
        file.on("limit", () => { return res.status(404).json({ general: "File size too large!" }); });
        file.on("end", () => { console.log(`Done processing [{filename}] ({mimetype})`); });

        const randomizedFileName = createFileName(filename);
        const filePath = path.join(os.tmpdir(), randomizedFileName);

        pendingFileWrites.push(createImagePromise(file, filePath, randomizedFileName, mimetype, fieldname));
    });
};

---

Note that you could make this async:

busboy.on("finish", () => {
    Promise.all(pendingFileWrites).then((fileWrites) => {
        // NOW we're done
        res.json({
            filesToUpload: fileWrites, 
            fields: fields
        });
    });
});

like

busboy.on("finish", async () => {
    var fileWrites = await Promise.all(pendingFileWrites);
    // NOW we're done
    res.json({
        filesToUpload: fileWrites, 
        fields: fields
    });
});

or even

busboy.on("finish", async () => {
    res.json({
        filesToUpload: await Promise.all(pendingFileWrites), 
        fields: fields
    });
});

if you wanted to. In either case you need to add error handling (the former through .catch(), the latter trough a try/catch block).