readFile synchronously nodejs

Question

I am new to nodejs and just started learning. I need to read 5 json files and place them in an array. I have created 2 functions: readDirectory and processFile.

    let transactionArray = [];
    router.get('/', (req,res) => {
    
    //joining path of directory 
    const directoryPath = path.join(__dirname, '../data');
    
    readDirectory(directoryPath);
    
    res.send(JSON.stringify(transactionArray))
    
    })

readDirectory will get the dir and will read the filenames.

    function readDirectory(directoryPath){
        //passsing directoryPath and callback function
    fs.readdir(directoryPath, function (err, files) {
        //handling error
        if (err) {
            return console.log('Unable to scan directory: ' + err);
        } 
        //listing all files using map
        let fileSummary = files.map(file => {
            
            //get the filename
            let categoryName = ''
    
            if (file.includes('category1')) {
                 categoryName = 'category1'
            } else if (file.includes('category2')) {
                 categoryName  = 'category2'
            } else {
                categoryName = 'Others'
            }
    
            // read the file
            const filePath = directoryPath +'/'+ file
    
            fs.readFile(filePath, 'utf8', (err, fileContents) => {
                if (err) {
                  console.error(err)
                  return
                }
                try {
                  let data = JSON.parse(fileContents, categoryName)
                  processFile(data, categoryName);
                   
                } catch(err) {
                    console.error(err)
                }
            })   
        }) 
    });    
    }

Then it will read the file using function processFile.

function processFile(data, categoryName)
{
    let paymentSource = ''

    if (categoryName == 'category1'){
       paymentSource = categoryName +': '+ categoryName +' '+ data.currency_code
    } else if (categoryName == 'category2') {
        paymentSource = categoryName +': '+ data.extra.payer +'-'+ data.currency_code 
    } else {
        paymentSource = 'Others'
    }
    
    let transactionDetails = new Transaction(
        data.id,
        data.description,
        categoryName,
        data.made_on,
        data.amount,
        data.currency_code,
        paymentSource)

    transactionArray.push(transactionDetails)
console.log(transactionArray);
}

The console log is something like this: [{Transaction1}] [{Transaction1},{Transaction2}] [{Transaction1},{Transaction2},{Transaction3}]

but the result on the UI is only []

During debug, I noticed that it is not reading synchronously so I tried using readFileSync but it did not work. How can I read both functions synchronously so it will not give an empty array?

Answer 1

Do some playing around to understand what the fs functions do when they have callbacks, and when they're synchronous. From the code that you have we have make a few changes so that you don't have to use the synchronous functions from the file system library.

First of all you need to wait for all the asynchronous tasks to complete before returning response.

router.get('/', async (req, res) => {
  // joining path of directory
  const directoryPath = path.join(__dirname, '../data')

  readDirectory(directoryPath).then(() => {
    res.send(JSON.stringify(transactionArray))
  }).catch(err => {
    res.status(500).json(err)
  })
})

Secondly, to keep the code as is as to teach you something about promises, lets wrap the first function in a promise.

function readDirectory (directoryPath) {
  return new Promise((resolve, reject) => {
    // passsing directoryPath and callback function
    fs.readdir(directoryPath, function (err, files) {
    // handling error
      if (err) {
        return console.log('Unable to scan directory: ' + err)
      }
      // listing all files using map
      const fileSummary = Promise.all(
       files.map(file => {
          return new Promise((resolve, reject) => {
          // get the filename
            let categoryName = ''

            if (file.includes('category1')) {
              categoryName = 'category1'
            } else if (file.includes('category2')) {
              categoryName = 'category2'
            } else {
              categoryName = 'Others'
            }

            // read the file
            const filePath = directoryPath + '/' + file

            fs.readFile(filePath, 'utf8', (err, fileContents) => {
              if (err) {
                console.error(err)
                reject(err)
              }
              try {
                const data = JSON.parse(fileContents, categoryName)
                processFile(data, categoryName).then(data => {
                  data()
                })
              } catch (err) {
                console.error(err)
                reject(err)
              }
            })
          })
        })
      ).then(() => {
        resolve()
      }).catch(err => {
        reject(err)
      })
    })
  })
}

Please refer to the bible (MDN) for javascript about promises -> https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise

And finally wrap the processFile function in a promise

function processFile (data, categoryName) {
  return new Promise((resolve, reject) => {
    let paymentSource = ''

    if (categoryName == 'category1') {
      paymentSource = categoryName + ': ' + categoryName + ' ' + data.currency_code
    } else if (categoryName == 'category2') {
      paymentSource = categoryName + ': ' + data.extra.payer + '-' + data.currency_code
    } else {
      paymentSource = 'Others'
    }

    const transactionDetails = new Transaction(
      data.id,
      data.description,
      categoryName,
      data.made_on,
      data.amount,
      data.currency_code,
      paymentSource)

    transactionArray.push(transactionDetails)
    console.log(transactionArray)
    resolve()
  })
}

What the heck am I doing? I'm just making your code execute asynchronous task, but wait for them to be completed before moving on. Promises are a way to handle this. You can easily pull this off with the FS synchronous functions, but this way you can learn about promises!