Chained Fetch Promises (JSON, React, Webpack, NodeJS)

Question

I need to fetch something 10 times and return the JSON of each request and push that into state. Most of the fetching works, but slows down about half way and halts before it completes. The promises seem to accept the 'cors' response, not the actual json, and thus triggers prematurely.

Also, I don't like how I have to repeat the same code so many times just to change the offset, but I spent hours fiddling with a for loop and I got stuck, so if you guys can suggest a better way to do this that would be awesome.

Here's the code:

function FetchAll(API_KEY, CX, query, E, that, pushState){

    fetch(`https://www.googleapis.com/youtube/v3/channels?part=contentDetails&id=${E.target.value}&key=${API_KEY}`, {
      method: 'get',
      headers : { 
        'Content-Type': 'application/json',
        'Accept': 'application/json'
       }
    }).then(function(response){
      return response.json()
    }).then(function(uploads){
        console.log(uploads)
      return fetch(`https://www.googleapis.com/youtube/v3/playlistItems?playlistId=${uploads.items[0].contentDetails.relatedPlaylists.uploads}&key=${API_KEY}&part=snippet&maxResults=50`)
    }).then(response => {
      return response.json()
    }).then(function(data){
        console.log(data)
      that.setState({yt:data})
    }).then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${1}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${11}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${21}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${31}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${41}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${51}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${61}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${71}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${81}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${91}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))
    .then(function(){
      return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=10&cx=${CX}&q=${query}&start=${101}`)
    }).then(function(response){
      return response.json();
    }).then(r => pushState(r))

}

export default FetchAll

In case you were wonering, FetchAll is being called in the main App.js file, with everything being sent to it in parameters.

This is pushState (if needed)

FetchAll(API_KEY, CX, query, e, that,
    (r) => {
      console.log(r)
      let c = that.state.compareRaw
      c.push(r)
      that.setState({
        compareRaw: c
      })
    }
    )
}

'that' is a reference to 'this'

Any help or tips are greatly appreciated. Thanks!

Answer 1

First decide which request are waterfall request and which requests are parallel requests.

In waterfall model current request is dependent from previous request result, and is handled with sequencing .then() functions

In parallel model request are independent and can all fire in the same time. It can be solved with bluebird as a great helper tool for promises.

const Promise = require('bluebird');
Promise.all([fetch(...), fetch(...)...., fetchN(...)], 
            (result1, result2, result3 ..., resultN) => {
              //handle results
            })

You can always wrap api call in a function:

function search(start, limit) {
    return fetch(`https://www.googleapis.com/customsearch/v1?key=${API_KEY}&num=${limit}&cx=${CX}&q=${query}&start=${start}`)  
}

All together now

fetch(...) // request 1
.then((response) => fetch(...)) // request 2
.then((response) => {
   const apiCalls = [];
   for let i = 0; i < 10; i++ {
      apiCalls.push( search(i*10+1, 10) );
   }
   return Promise.all(apiCalls);
})
.then((...results) => {
   // handle search results
})

Hope it helps.