I want to create a new Promise inside an async function. Inside this Promise, i'd like to call another async function which will call some async stuff (scrape the HTML from a web page).
Here's my code :
async extractProductUrlFromHTML(html) {
const products = html(`h2.product-name`).toArray();
let extracted = [];
let promises = [];
for (let index = 0; index < products.length; index++) {
const element = products[index];
let productUrl = element.children[0].attribs.href;
const productProm = new Promise(async (resolve, reject) => {
try {
console.log("process");
const product = await this.getProductInfos(productUrl);
console.log("resolving")
resolve(product);
} catch (error) {
reject(error)
}
})
promises.push(productProm);
}
Promise.all(promises).then((prods) => {
console.log("promises ok", prods)
extracted.push(prods);
return (extracted);
})
}
And here's how the function is called :
async scrapeAllProducts({ request, response }) {
let extractedProducts = []
//get the html page
const html = await AxiosService.getHTML('http://www.ubagcollection.com/fr/ubag.html?limit=36')
const $ = cheerio.load(html);
let pagesToScrape = await this.extractPagination($);
pagesToScrape.push(`http://www.ubagcollection.com/fr/ubag.html?limit=36`)
for (let index = 0; index < pagesToScraep.length; index++) {
const url = pagesToScrape[index];
let html = await AxiosService.getHTML(url);
const $ = cheerio.load(html);
console.log("start extraction")
const products = await this.extractProductUrlFromHTML($);
console.log('product extracted from page :' + index)
extractedProducts.push(products);
}
console.log("all is extracted", extractedProducts)
... bla bla bla
My problem is that at the moment all my promises are created, I can see the "product extraced from page : ... ", way before the program finished to work...
here's the sequence I can see when I run my script :
start extraction process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process process product extracted from page :1
each time a promise is created, I log 'process', and I should see the "resolving" log when my async function has finished her job, which is not the case here.
Ok I fixed my problem. I drop a code sample, maybe it might be useful for someone.
let extracted = [];
let promises = [];
for (let index = 0; index < products.length; index++) {
const element = products[index];
let productUrl = element.children[0].attribs.href;
const productProm = new Promise(async (resolve, reject) => {
try {
await utils.timeout(utils.getRandomInt(1000,4000))
const product = await this.getProductInfos(productUrl);
resolve(product);
} catch (error) {
reject(error)
}
})
promises.push(productProm);
}
let prods = await Promise.all(promises)
extracted.push(prods);