Async and recursive directory scan, for file listing in Nodejs and Expressjs

Question

In this Expressjs route file I'm trying to get (recursively) all the JSON files inside a ./data directory.

Actually I can console.log the file ehere you can see the A Mark, but I can't find the way to send the whole complete bunch of paths to the view once the async stuff finalized.

Some help would be really appreciated.

This is the data ./data structure:

--- dir1
    `-- json1.json
    `-- json2.json
--- dir2
    `-- json3.json
--- dir3

const express = require('express'),
    router = express.Router(),
    fs = require('fs'),
    path = require('path')
    ;

let scan = function (directoryName = './data') {

    return new Promise((resolve, reject) => {

        fs.readdir(directoryName, function (err, files) {
            if (err) reject(err);

            files.map((currentValue, index, arr) => {
                let fullPath = path.join(directoryName, currentValue);

                fs.stat(fullPath, function (err, stat) {
                    if (err) reject(err);

                    if (stat.isDirectory()) {
                        scan(fullPath);
                    } else {
                        console.log(currentValue); <= (A mark)
                        //resolve();
                    }
                });
            });
        });
    })
};


router.get('/', (req, res, next) => {
  scan()
        .then(data => res.render('list', {
            title: 'List',
            data: data
        }))
        .catch(next);
});

module.exports = router;

Answer 1

You can simplify the task a bunch if you promisify the fs functions you're using so that all async logic is promises and then use async/await to help you serialize the flow of control.

Here's one way to do that:

const promisify = require('util').promisify;
const path = require('path');
const fs = require('fs');
const readdirp = promisify(fs.readdir);
const statp = promisify(fs.stat);

async function scan(directoryName = './data', results = []) {
    let files = await readdirp(directoryName);
    for (let f of files) {
        let fullPath = path.join(directoryName, f);
        let stat = await statp(fullPath);
        if (stat.isDirectory()) {
            await scan(fullPath, results);
        } else {
            results.push(fullPath);
        }
    }
    return results;
}

The above code was tested in node v10.14.1.

You could then use that the same way you were:

router.get('/', (req, res, next) => {
  scan().then(data => res.render('list', {
      title: 'List',
      data: data
   })).catch(next);
});

---

FYI, there is a newer (still experimental) promise-based API for the fs module. You can use that like this:

const path = require('path');
const fsp = require('fs').promises;

async function scan2(directoryName = './data', results = []) {
    let files = await fsp.readdir(directoryName, {withFileTypes: true});
    for (let f of files) {
        let fullPath = path.join(directoryName, f.name);
        if (f.isDirectory()) {
            await scan2(fullPath, results);
        } else {
            results.push(fullPath);
        }
    }
    return results;
}

Note, this new version also uses the new withFileTypes option that saves having to call stat() on every file.