JS async queue with return data

Question

I want to make asynchronous requests to the google api and save the results in one array. My attempts to somehow return a value from the apiHandler have failed. I can't figure uot where I need to grab thsi data. Can you help me?

I'm not very experienced with js, much less asynchronous js.

import { queue } from "async";
import fetch from "node-fetch";

let concurrency = 10; // How many async processes will be running
let KEY = '';
let ids = [
    'sMTs5bvAsGA',
    'kW5X4dU0gnY',
    'dX2hyyKFIHA',
    '91egEkKuVas',
    'DfhKbYXRu6Q',
    'hQdOqC-1tVg',
    'AOwFmabnb7s',
];

let data = [];

export const taskQueue = queue(async (task, done) => {
    try {
        let data = await task();
        done;
        return data;
    } catch (err) {
        throw err;
    }
}, concurrency);

taskQueue.drain(() => {
    console.log("All items completed!\n");
    process.exit();
});

export function getItems(video_ids) {
    video_ids.forEach((id) => {
        taskQueue.push(() =>
            apiHandler(`https://www.googleapis.com/youtube/v3/search?part=snippet&relatedToVideo=${id}&maxResults=50&type=video&key=${KEY}`),

            (err) => {
                if (err) {
                    console.log(err);
                    throw new Error('Error getting data.');
                }
            })
    });
};

async function apiHandler(url) {
    const response = await fetch(url);
    const data = await response.json();
    return data;
}

getItems(ids);

P.S Different implementation

import fetch from "node-fetch";

let concurrency = 10; // How many async processes will be running
let KEY = '';
let video_ids = [
    'sMTs5bvAsGA',
    'kW5X4dU0gnY',
    'dX2hyyKFIHA',
    '91egEkKuVas',
    'DfhKbYXRu6Q',
    'hQdOqC-1tVg',
    'AOwFmabnb7s',
];

let data = [];
let promises = [];

function apiHandler(url) {
    return fetch(url).then((response) => {
        data.push(response.json);
    });
}

video_ids.forEach((id) => {
    promises.push(() =>
        apiHandler(`https://www.googleapis.com/youtube/v3/search?part=snippet&relatedToVideo=${id}&maxResults=50&type=video&key=${KEY}`),

        (err) => {
            if (err) {
                console.log(err);
                throw new Error('Error getting data.');
            }
        })
});

Promise.all(promises).then((response) => {
    console.log(data)
});

Answer 1

For your problem it seems that it makes sense to build an array of Promise elements and then call .all().

A Promise is an object which can be resolved or rejected. Assuming that your fetch is a promise, you can do the following:

1. Create an empty array

Somewhere you need to create an empty array that will have to be accessed in the relevant places of your algorithm. For the sake of simplicity, I will create it as a variable:

let promises = [];

2. Make sure apiHandler returns a promise

function apiHandler(url) {
    return fetch(url).then((response) => {
        data.push(response);
    });
}

3. At your loop hydrate your promise array

    video_ids.forEach((id) => {
        promises.push(
            apiHandler(`https://www.googleapis.com/youtube/v3/search?part=snippet&relatedToVideo=${id}&maxResults=50&type=video&key=${KEY}`)
        )
    });

4. Somewhere, possibly just after your loop, create an .all() handler

Promise.all(promises).then((response) => {
    //do something
});

Explanation

The .then() and .all() calls also create promises and you pass an array of promises to .all(), so its .then() will be called when all promises were fulfilled. As about promises, it was built of the .then() handlers of your fetch calls.

EDIT

Proof-of-concept

let data = [];
let promises = [];
for (let i = 0; i < 10; i++) {
    promises.push(new Promise((resolve, reject) => setTimeout(function() {
         data.push(i);
         resolve(i);
    }, i * 100)));
};
console.log('FIRST OUTPUT ' + JSON.stringify(data)); //empty
Promise.all(promises).then(() => {
    console.log('THIRD OUTPUT ' + JSON.stringify(data));
});
console.log('SECOND OUTPUT ' + JSON.stringify(data)); //empty