Search code examples
javascriptnode.jspromisees6-promise

Node - wait for map to finish before continuing


I have this file in my node app that supposed to go fetch me some data about every league champion from their official website using cheerio and its going all great but when I add all the data to my array to then return it as json data the write function runs before the map finishes so I just creating a json file with an empty array in it:

const request = require('request');
const cheerio = require('cheerio');
const fs = require('fs');

const champions = fs.readFileSync('champions.json');
const championsObj = JSON.parse(champions);

let champsList = [];

championsObj.map(champ => {
  request(champ.href, (err, res, html) => {
    if (!err && res.statusCode == 200) {
      const $ = cheerio.load(html);

      const champName = $('.style__Title-sc-14gxj1e-3 span').text();

      let skins = [];

      const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
        (i, el) => {
          const skinName = $(el).text();
          skins.push = skinName;
        }
      );

      const champion = {
        champName,
        skins
      };

      console.log(champion);

      champsList.push = champion;
    }
  });
});

const jsonContent = JSON.stringify(champsList);

fs.writeFile('champions2.json', jsonContent, 'utf8', function(err) {
  if (err) {
    console.log(err);
  }
});

I'm not a node expert but I tried using Promise but it didn't work but I'm not sure maybe I used it wrong.

UPDATE #1: using axios

championsObj.map(async champ => {
  const html = await axios.get(champ.href);
  const $ = await cheerio.load(html);

  const champName = $('.style__Title-sc-14gxj1e-3 span').text();

  let skins = [];

  const skinsList = $('.style__CarouselItemText-sc-1tlyqoa-16').each(
    (i, el) => {
      const skinName = $(el).text();
      skins.push = skinName;
    }
  );

  const champion = {
    champName,
    skins
  };

  console.log(champion);

  champsList.push = champion;
});

Solution

  • Your problem here is that Array#map doesn't wait for asynchronous functions such as the request calls to finish before moving on. I recommend p-map with got. To ensure perfect execution order, I also recommend reading and writing the file asynchronously.

    const got = require('got');
    const pMap = require('p-map');
    const cheerio = require('cheerio');
    const fs = require('fs').promises;
    
    (async () => {
        const champions = JSON.parse(await fs.readFile('champions.json', 'utf8'));
    
        let champsList = await pMap(champions, async champ => {
            const {
                body
            } = await got(champ.href)
    
            const $ = cheerio.load(body);
    
            const champName = $('.style__Title-sc-14gxj1e-3 span').text();
    
            let skins = [];
    
            $('.style__CarouselItemText-sc-1tlyqoa-16').each(
                (_, el) => {
                    const skinName = $(el).text();
                    skins.push(skinName);
                }
            );
    
            const champion = {
                champName,
                skins
            };
    
            console.log(champion);
    
            return champion;
        })
    
        await fs.writeFile('champions2.json', JSON.stringify(champsList));
    })();