How to push the data returned by a promise into an array?

Question

I am making a call to Udemy API. To make simultaneous calls, I am using a loop. By doing so, I am automatically incrementing the page numbers and trying to fetch the data from each page and store it into an array so that I can write all the data into a single file in json format. But all I am getting is an empty array. How do I access the value returned by the promise and store into the doc.table array?

My code:

const fetch=require("node-fetch");
const fs=require("fs");
let doc={};
doc.table=[];

for(let i=1;i<=10;i++){

fetch('https://www.udemy.com/api-2.0/courses/ page='+i+'&page_size=10&client_id=${client_id}&client_secret=${client_secret},{
      method:'GET',
      body:null,
      headers:{authorization: ${auth_code}}
      })
      .then(res=>res.json())
      .then(json=>doc.table.push(json))
};


fs.writeFile("UDEMY.json",JSON.stringify(doc),function(err){
    if(err) throw err;
    console.log("Complete");
});

Answer 1

I'd suggest using await so that your for loop will be paused for each iteration:

const fetch = require("node-fetch");
const fsp = require("fs").promises;

let doc = { table: []};

async function run() {
    for (let i = 1; i <= 10; i++) {

        let data = await fetch(`https://www.udemy.com/api-2.0/courses?page=${i}&page_size=10&client_id=${client_id}&client_secret=${client_secret}`,{
              method:'GET',
              body:null,
              headers:{authorization: auth_code}
        }).then(res=>res.json());

        doc.table.push(data);
    }

    await fsp.writeFile("UDEMY.json",JSON.stringify(doc));
    console.log("done");
}

run().catch(err => {
    console.log(err);
});

The other possibility is to run all the requests in parallel and use Promise.all() to know when they are all done. They key to both solutions is to use the promise that fetch() returns to control knowing when things are done.

If you really want to run them in parallel and you're sure that your target host will allow it, you can do this:

const fetch = require("node-fetch");
const fsp = require("fs").promises;

let doc = { table: []};

function run() {
    let promises = [];
    for (let i = 1; i <= 10; i++) {

        promises.push(fetch(`https://www.udemy.com/api-2.0/courses?page=${i}&page_size=10&client_id=${client_id}&client_secret=${client_secret}`},{
              method:'GET',
              body:null,
              headers:{authorization: ${auth_code}}
        }).then(res=>res.json()));

    }
    return Promise.all(promises).then(data => {
        doc.table = data;
        return fsp.writeFile("UDEMY.json",JSON.stringify(doc));
    });

}

run().then(() => {
    console.log('done');
}).catch(err => {
    console.log(err);
});

And, if you want some level of parallel requests, but want to limit how many are in parallel, you can use mapConcurrent() described here.