Search code examples
javascriptpromisees6-promisep-limit

How to use pendingCount in p-limit?


As I'm doing a lot of calls to an API I wanted a way to limit the amount of calls I do concurrently.

I found p-limit to work perfectly for my needs https://www.npmjs.com/package/p-limit

Now, I want to output the current progress to the user. I see this library has a pendingCount property, but I'm clueless on how to use it.

Any help will be appreciated.

There is a similar question in How to use the limit.pendingCount variable in p-limit? but the edited answer is not using pendingCount

I want to be able to write "we are still missing n API calls"

This is my current code (simplified):

import pLimit from 'p-limit';

const aPromise = [];
const urls = [
"https://www.google.com/search?q=1",
"https://www.google.com/search?q=2",
"https://www.google.com/search?q=3",
"https://www.google.com/search?q=4",
"https://www.google.com/search?q=5",
"https://www.google.com/search?q=6",
"https://www.google.com/search?q=7",
"https://www.google.com/search?q=8"
];

const limit = pLimit(3);

for (const url of urls) {

    try {
        aPromise.push(limit(() => fetch(url)));
    } catch (error) {
        console.error('An error occurred:', error);
    }

}

const res = await Promise.allSettled(aPromise);

for (const x of res) {
    if (x.status == "fulfilled") {
        //process results
    } else if (x.status == "rejected") {
        console.log(`${x.reason}`);
    }
}

Adding solution, Thanks Simon

import pLimit from 'p-limit';

const aPromise = [];
const urls = [
"https://www.google.com/search?q=1",
"https://www.google.com/search?q=2",
"https://www.google.com/search?q=3",
"https://www.google.com/search?q=4",
"https://www.google.com/search?q=5",
"https://www.google.com/search?q=6",
"https://www.google.com/search?q=7",
"https://www.google.com/search?q=8"
];

const limit = pLimit(3);

for (const url of urls) {

    try {
        aPromise.push(limit(() => fetch(url)));
    } catch (error) {
        console.error('An error occurred:', error);
    }

}

let nProgressInterval = setInterval( () => {
    printProgress(Math.round(100 - limit.pendingCount * 100 / aPromise.length))
}, 1000);

const res = await Promise.allSettled(aPromise);

clearInterval(nProgressInterval);
nProgressInterval = null;

for (const x of res) {
    if (x.status == "fulfilled") {
        //process results
    } else if (x.status == "rejected") {
        console.log(`${x.reason}`);
    }
}

// This will write the progress number in the same position
function printProgress(progress) {
    process.stdout.clearLine(0);
    process.stdout.cursorTo(0);
    process.stdout.write(`Progreso ${progress}%`);
}

Solution

  • Adding solution, Thanks Simon

    
    import pLimit from 'p-limit';
    
    const aPromise = [];
    const urls = [
    "https://www.google.com/search?q=1",
    "https://www.google.com/search?q=2",
    "https://www.google.com/search?q=3",
    "https://www.google.com/search?q=4",
    "https://www.google.com/search?q=5",
    "https://www.google.com/search?q=6",
    "https://www.google.com/search?q=7",
    "https://www.google.com/search?q=8"
    ];
    
    const limit = pLimit(3);
    
    for (const url of urls) {
    
        try {
            aPromise.push(limit(() => fetch(url)));
        } catch (error) {
            console.error('An error occurred:', error);
        }
    
    }
    
    // Start an interval that every one secnd will print progress
    let nProgressInterval = setInterval( () => {
        printProgress(Math.round(100 - limit.pendingCount * 100 / aPromise.length))
    }, 1000);
    
    const res = await Promise.allSettled(aPromise);
    
    // Kill interval after all settled
    clearInterval(nProgressInterval);
    nProgressInterval = null;
    
    for (const x of res) {
        if (x.status == "fulfilled") {
            //process results
        } else if (x.status == "rejected") {
            console.log(`${x.reason}`);
        }
    }
    
    // This will write the progress number in the same position
    function printProgress(progress) {
        process.stdout.clearLine(0);
        process.stdout.cursorTo(0);
        process.stdout.write(`Progreso ${progress}%`);
    }