Search code examples
javascriptnode.jsforeachxmlhttprequest

XMLHTTPRequest inside forEach loop doesn't work


Hi I'm writing a short node.js app that sends an XMLHTTPRequest to an API each time it iterates over an array. The issue is it continues the foreach loop before the request is returned because of the async nature. I'm probably overlooking something big but I've spent the better part of this afternoon trying to rack my brain over this. I've tried using await to no avail, any solutions would be appreciated.

Thanks in advance.

NODE JS app

const mongoose = require("mongoose");
const fs = require("fs");
const ajax = require("./modules/ajax.js");

// Bring in Models
let Dictionary = require("./models/dictionary.js");


//=============================
//     MongoDB connection
//=============================

// Opens connection to database "test"
mongoose.connect("mongodb://localhost/bookCompanion");
let db = mongoose.connection;

// If database test encounters an error, output error to console.
db.on("error", (err)=>{
  console.console.error("Database connection failed.");
});

// Check for connection to the database once.
db.once("open", ()=>{
  console.info("Connected to MongoDB database...");

  fs.readFile("./words.json", "utf8", (err, data)=>{

    if(err){
      console.log(err);
    } else {
      data = JSON.parse(data);
      data.forEach((word, index)=>{

        let search = ajax.get(`LINK TO API?=${word}`);

        search.then((response)=>{

          let newWord = new Dictionary ({
            Word: response.word,
            phonetic: response.phonetic,
            meaning: response.meaning
          }).save();

          console.log(response);

        }).catch((err)=>{
          console.log(err);
        });

      });
    }

  });


});

XMLHTTPRequest Module

// Get Request module utilising promises

const XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest;

const get = (url)=>{
  // This function will return a promise, promises use resolve and reject. The resolve is accessed through .then and the reject through the .catch
  return new Promise((resolve, reject)=>{

    // Create new XMLhttp (AJAX) Request
    let xhr = new XMLHttpRequest();
    // Sets up the request, setting it to a GET request, pointing to the website and setting it to asynchronous
    xhr.open("GET", url , true);
    //sends the request
    xhr.send();

    xhr.onload = ()=>{
      if (xhr.status == 200){
        // When loaded pass the response over to the .then method
        resolve(JSON.parse(xhr.responseText));
      } else {
        // if error pass the status code to the .catch method for error handling
        reject(xhr.statusText);
      }
    };

    xhr.onerror = ()=>{
      // if error pass the status code to the .catch method for error handling
      reject(xhr.statusText && xhr.status);
    };

  });
};

module.exports.get = get;

Solution

  • It seems my code works fine when working with smaller arrays, the real issue I'm having is dealing with the blocking nature of the forEach loop and memory. The array I need to loop through composes of over 400,000+ words and the app runs out of memory before the forEach loop can finish and free up the call stack for the httprequests to resolve.

    Any information on how I can create a synchronous forEach loop that doesn't block the call stack would be very appreciated.