How to use multiple time web scraping in Dialogflow fulfillment inline? and how to detect error at fulfillment inline code?

Question

I have some problem about using Dialogflow fulfillment inline for multiple web scraping.

Below are some parts of code that relate with web scraping.

When I can request 1st web scraping only (not call 2nd scraping by comment 2nd '.then'), it is success to retrieve 'targetUrl' from 1st scraping. And it can this display at Response Message on Dialogflow.

But When I use 2nd scraping, 'targetUrl' from 1st scraping cannot display at Response Message on Dialogflow. The Response Message is message, which set at Intent Response section. But messages are added at Fulfillment did not appear.

I don't know what is happened. So my questions are

1. How to use multiple time web scraping in Dialogflow fulfillment inline?

2. How to detect error at fulfillment inline code?

Thanks in advance

index.js

const functions = require('firebase-functions');
const {WebhookClient} = require('dialogflow-fulfillment');
const {Card, Suggestion} = require('dialogflow-fulfillment');
const {Payload} = require('dialogflow-fulfillment');
const rp = require('request-promise-native');
const rp2 = require('request-promise-native');
const cheerio = require('cheerio');
.....

exports.dialogflowFirebaseFulfillment = functions.https.onRequest((request, response) => {
    .....
    .....
    function loadElements(data) {  // --> function for load data from 2nd web scraping
        $ = cheerio.load(data);
        resultPrizeElems = $('td.border-0');
    }

    function gathering2WebScrapList(agent) {  // --> function for 2nd web scraping
        return rp2(targetUrl).then((data) => {  // -- Request 2nd web scraping use 'targetUrl value from 1st web scraping
            loadElements(data);   // cheerio load result from 2nd web scraping
            agent.add(resultPrizeElems.length.toString());  // --> Add array size to 'agent' for checking but nothing happen at Dialogflow Response
            return Promise.resolve(agent);
        }).catch((error) => {
            console.log(error);
        });
    }

    function loadTargetDateElements(data) {   // --> function for load data from 1st web scraping --> 'success'
        $ = cheerio.load(data);
        targetDateElems = $('div.radio');
    }

    function load1stWebScrapData(agent) {
        return rp(`http://www.firstdomain.com/page.aspx`).then((data) => { // 'Success' to 1st web scraping
            loadTargetDateElements(data);    // use cheerio for load html element success
            .... 
            targetUrl = assign value from get something when 1st web scraping
            ....
            agent.add(targetUrl);  // add 'targetUrl' value to display on Response Message 'OK'
            return Promise.resolve(agent);
        }).then((agent) => {  // Second 'then'
            return gathering2WebScrapList(agent); // --> function call request 2nd web scraping
        }).catch((error) => {
            console.log(error);
        });
    }

    intentMap.set('Load Data', load1stWebScrapData);
    agent.handleRequest(intentMap);
}

package.json

{
  "name": "dialogflowFirebaseFulfillment",
  "description": "This is the default fulfillment for a Dialogflow agents using Cloud Functions for Firebase",
  "version": "0.0.1",
  "private": true,
  "license": "Apache Version 2.0",
  "author": "Google Inc.",
  "engines": {
    "node": "10"
  },
  "scripts": {
    "start": "firebase serve --only functions:dialogflowFirebaseFulfillment",
    "deploy": "firebase deploy --only functions:dialogflowFirebaseFulfillment"
  },
  "dependencies": {
    "actions-on-google": "^2.12.0",
    "cheerio": "^1.0.0-rc.3",
    "iconv-lite": "^0.6.0",
    "firebase-admin": "^8.12.1",
    "firebase-functions": "^3.7.0",
    "dialogflow": "^1.2.0",
    "dialogflow-fulfillment": "^0.6.1",
    "request": "^2.88.2",
    "request-promise-native": "^1.0.8"
  }
}

Answer 1

To debug the inline editor code you have to use the Google Stackdriver from the Google Cloud console.