slowness in promise chain

Question

my codes transfers huge number of files from one cloud storage to another cloud storage (same region). The workflow is downloading the source file stream, then uploading the stream to the target storage.

If running them in a non-promise loop, the transferring is fast(say, 100M/s), but will hit the memory limit. Finally the server crashes.

if in a promise chain, i.e. run the next job after the last job completes, the crash problem is solved, but the transferring speed is very slow (say 10M/s).

My question: why the promise would affect the downloading & uploading speed? or anything I missed?

code snippet:

transferArray.forEach(function (eachTransfer) { 
      queue = queue.then(function(result){
        // put result somewhere
        return eachFileTransfer(jobId,userid,eachTransfer);
      });  
    }); 
    queue.then(function(){
      console.log('done');
    });

I am thinking to use PromisePool with concurrences, but not sure how much the speed would be improved, and the reasonable number of concurrences I should set. the ref post: Execute promises concurrently with a buffer pool size in Javascript

Answer 1

finally, I made it working by PromisePool. The code snippet below is based on the other post: Execute promises concurrently with a buffer pool size in Javascript

Since my code is relatively complex than this post, thought it might be helpful for some others:

var PromisePool = require('es6-promise-pool'); 

//single promises
function p1(){
  return new Promise(function(resolve, reject) {
      console.log('p1');  
      setTimeout(resolve, 2000, 'foo');
 });
}
function p2(){
  return new Promise(function(resolve, reject) {
      console.log('p2');  
      setTimeout(resolve, 2000, 'foo');
  });
}
function p3(){
 return new Promise(function(resolve, reject) {
    console.log('p3');  
    setTimeout(resolve, 2000, 'foo');
 });
}


var tasks=[];
var loopIndex = 0;
[1,2,3,4,5,6,7,8].forEach(function(v,i){

   console.log(v,i);

   //build promise chain
   var x = (v) => new Promise(function(resolve, reject) {
      console.log(v);

      p1().then(function(r){
         return p2();
      })
      .then(function(r){
        return p3();
      })
      .then(function(r){
        //console.log('embedded chain done');
        resolve('embedded chain done');
       }).catch(function(e){
        reject('embedded chain failed');
     }); 
    //setTimeout(resolve, 2000, 'foo');
  })

  //build one promise task
  tasks.push({fun:x,param:i});

  if(++loopIndex == 8){
     //once the array is done
    const promiseProducer = () => {
        while(tasks.length) {
           console.log('processing ');
           const task = tasks.shift();
           return task.fun(task.param);  
        } 
        return null;
     }

      // concurrent Promises set to 3
      const pool = new PromisePool(promiseProducer, 3); 
     //start to promise all, yet with concurrent 3 tasks.
     const poolPromise = pool.start(); 
     poolPromise.then(() => { console.log('done!'); })
    }  

});