Search code examples
javascriptgoogle-apps-scripturlfetch

How to pull data from additional pages through pagination


I successfully returned the first page of data and have the number of additional pages of data that exist in the API call.

This is the code that I have tried to pull the additional pages of data.

try {
const response = UrlFetchApp.fetch(root + endpoint, params);
const responseCode = response.getResponseCode();
const returnedResults = response.getContentText();
const jsonResults = JSON.parse(returnedResults)
//console.log(responseCode) // *** return 200
//console.log(returnedResults) // *** return successful
//console.log(jsonResults) //*** return successful


Object.keys(jsonResults).forEach(function (key){
  console.log(jsonResults[key]);
  /*
  { count_per_page: 20,
    total_count: 12261,
    current_page: 1,
    total_pages: 614,
    _links: { next: { href: '/v2/organizations?page=2' } } }
  */

  });
} catch (e) {
  console.log('Error: ' + e);
  }

const data = [];

let hasmoreitems = true;
  while (hasmoreitems) {
    
    hasmoreitems = jsonResults.pagination.current_page < jsonResults.pagination.total_pages;
   
  data.forEach(entry => {
    const name = entry['name'];
    const email = entry['email'];
    const mission = entry['mission_statement'];
    const address = entry['address'].country;

  data.push([
    name, email, mission, address]);
  });

  // send data to spreadsheet
  const sheet = SpreadsheetApp.openById('1fM-soJt8rJeIHYZ2vVKyoX9J-LohElqvGB2syhlRk00').getSheets()[1];

  sheet.getRange(sheet.getLastRow()+1, 1, data.length,data[0].length).setValues(data);
  }

A sample response for pagination would look like this;

"pagination": {
        "count_per_page": 20,
        "total_count": 1,
        "current_page": 1,
        "total_pages": 2,
        "_links": {
            "next": {
                "href": "/v2/organizations?page=2"
            }
        }
    }

Solution

  • After examining the structure of the object, I realized the different levels of the keys. Below is the working code. Hope that helps.

    let all_data = [];
    
      while (endpoint) {
        let response = UrlFetchApp.fetch(root + endpoint, params);
        let data = JSON.parse(response.getContentText());
        let orgs = data.organizations
    
        Object.keys(data).forEach(function (key) {
          console.log(data[key]);
        });
    
        endpoint = data.pagination._links.next.href;
        orgs.forEach((entry) => {
          all_data.push([
            entry.name,
            entry.email,
            entry.mission_statement,
            entry.address.country,
          ]);
          //console.log(all_data)
        });
           
     }