Nodejs, How to copy several file in nodejs without crash

Question

I tried to copy several file with node js.

Here is my an example of what i'm trying to do :

var request = require('request');
 va photos [{ 'url': 'http://xxxx.com/im1', 'name' : 'name1' }, { 'url': 'http://xxxx.com/im12', 'name' : 'name2'    }, 
 for (var i = 0; i < photos.length; i++) {
                        request(photos[i].source).pipe(fs.createWriteStream(photos[i].name));
  }

After maybe 1000 call i have a socket hang out error.

Following @Timothy Strimple advice i decided to use async module.

My code is now something like this :

async.whilst(function () { return !stop; },
                function (callback) {
                    console.log("get next 20 image");
                    JM.api('/' + album.id + '/photos', { after: next }, function (resf) {
                        if (!resf || resf.error) {
                            console.log(!resf ? 'error occurred' : resf.error);
                        }
                        console.log("albums" + album.id + " " + resf.data.length + " dir" + dir);

                        async.eachSeries(resf.data, function (photo, done) {

                            request(photo.source).pipe(fs.createWriteStream(dir + "/" +photo.name));
                            console.log("copy of image " + img_basename);
                        }, function (err) {
                            if (err) {
                                console.log('An images failed to copy');
                            } else {
                                console.log('All 20 image have been copied successfully');
                            }
                            if (resf.paging && resf.paging.cursors) {
                                console.log("suite de l'album à venir");
                                next = resf.paging.cursors.after;
                                setTimeout(function () { callback(); }, 5000);
                            }
                            else {
                                console.log("Fin de l'album");
                                stop = true;
                                setTimeout(function () { callback(); }, 5000);
                            }
                        });
                    });
                },
                function (err) {
                    if (err) {
                        console.log('An images failed to process');
                        albumcallback();
                    } else {
                        console.log('All images in this group have been processed successfully');
                        albumcallback();
                    }
                }
            );// end while

I still having a crash after maybe 1 00 file copied. I'm sure that async.whilst and async.eachSeries are weel because my log show that each call is on series. But i have a crash. I temporary solved the proble by ading a wait after each copy like this :

request(photo.source).pipe(fs.createWriteStream(dir + "/" + img_basename));
                            console.log("copy of image " + img_basename);
                            setTimeout(function () { done(); }, 5000);

Is it a limit of request module ? How to change this fea line to make sure that each connection are closed before continung the program ?

Answer 1

The request call and pipe call are asyncrhon. So i have to rewrite this line : request(photos[i].source).pipe(fs.createWriteStream(photos[i].name));

See here : Downloading N number of remote files using Node.js synchronously