wait for writestream to finish before executing next function

Question

I have two functions. The first function reads all the files in a folder and writes their data to a new file. The second function takes that new file (output of function 1) as input and creates another file. Therefore it has to wait until the write stream of function 1 has finished.

const fs = require('fs');
const path = require('path');

function f1(inputDir, outputFile) {
    let stream = fs.createWriteStream(outputFile, {flags:'a'}); // new data should be appended to outputFile piece by piece (hence flag a)
    let files = await fs.promises.readdir(inputDir);

    for(let file of files) {
        let pathOfCurrentFile = path.join(inputDir, file);
        let stat = fs.statSync(pathOfCurrentFile);
        if(stat.isFile()) {
            data = await fs.readFileSync(pathOfCurrentFile, 'utf8');

            // now the data is being modified for output
            let result = data + 'other stuff';
            stream.write(result);
        }
    }
    stream.end();
}

function f2(inputFile, outputFile) {
    let newData = doStuffWithMy(inputFile);
    let stream = fs.createWriteStream(outputFile);
    stream.write(newData);
    stream.end();
}

f1('myFiles', 'myNewFile.txt');
f2('myNewFile.txt', 'myNewestFile.txt');

Here's what happens:

• 'myNewFile.txt' (output of f1) is created correctly
• 'myNewestFile.txt' is created but is either empty or only contains one or two words (it should contain a long text)

When I use a timeout before executing f2, it works fine, but I can't use a timeout because there can be thousands of input files in the inputDir, therefore I need a way to do it dynamically.

I've experimented with async/await, callbacks, promises etc. but that stuff seems to be a little to advanced for me, I couldn't get it to work.

Is there anything else I can try?

Answer 1

Since you asked about a synchronous version, here's what that could look like. This should only be used in a single user script or in startup code, not in a running server. A server should only use asynchronous file I/O.

// synchronous version
function f1(inputDir, outputFile) {
    let outputHandle = fs.openSync(outputFile, "a");
    try {
        let files = fs.readdirSync(inputDir, {withFileTypes: true});
        for (let f of files) {
            if (f.isFile()) {
                let pathOfCurrentFile = path.join(inputDir, f.name);
                let data = fs.readFileSync(pathOfCurrentFile, 'utf8');
                fs.writeSync(outputHandle, data);
            }
        }
    } finally {
        fs.closeSync(outputHandle);
    }
}

function f2(inputFile, outputFile) {
    let newData = doStuffWithMy(inputFile);
    fs.writeFileSync(outputFile, newData);
}

f1('myFiles', 'myNewFile.txt');
f2('myNewFile.txt', 'myNewestFile.txt');