How do I access the scraped data from node module in my html code in NodeWebkit

Question

I am trying to create an app using the NodeWebkit. I am scraping content using node-phantom-simple module. Using the module, I was able to scrape content off the website. But how should i access it on the html side. I don't think i can create a rest service for this case. Here is the code sample:

var file = require('file.js');
var gui = require('nw.gui');

var menu = new gui.Menu({ type: 'menubar' });

var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app     = express();

var driver = require('node-phantom-simple');



url = 'http://www.espncricinfo.com/';

request(url, function(error, response, html){
    if(!error){
        var $ = cheerio.load(html);

        var title, release, rating;
        var json = { title : "", release : "", rating : ""};

        $('.scoreline-list').first().filter(function(){
            var data = $(this);
            var numOfMatches = data.children().length;
            console.log("Number of Matches:  ",numOfMatches);
            var matches=[];

            //GET URL FOR EACH MATCH
            for(x=0;x<numOfMatches;x++)
            {
                var lielem = data.children().eq(x);
                matches[x] = "http://www.espncricinfo.com" + lielem.children().first().attr('href');
                $('#editor').val(matches[x]);
                console.log(matches[x]);

            }

            //FOR EACH MATCH URL
            for(x=0;x<numOfMatches;x++)
            {  
                var matchurl = matches[x];   
                //console.log(matchurl);
                driver.create({ path: require('phantomjs').path }, function (err, browser) {
                  return browser.createPage(function (err, page) {
                    return page.open(matchurl, function (err,status) {
                      console.log("opened site? ", status);
                      page.includeJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function (err) {
                        // jQuery Loaded. 
                        // Wait for a bit for AJAX content to load on the page. Here, we are waiting 5 seconds. 
                        setTimeout(function () {
                          return page.evaluate(function () {
                            //Get what you want from the page using jQuery. A good way is to populate an object with all the jQuery commands that you need and then return the object. 
                            var h2Arr = [];

                            $('.innings-information').each(function () { h2Arr.push($(this).html()); });

                            return {
                              h2: h2Arr
                            };
                          }, function (err,result) {
                            console.log(result);
                            browser.exit();
                          });
                        }, 5000);
                      });
                      });
                  });
                });
             setTimeout(function(){
                    //waiting for the jquery to load
                }, 5000);

            } //END FOR LOOP EACH MATCH URL




        })
    }
})

Thanks for the help!

Answer 1

Why wouldn't you be able to make a rest endpoint for this? Just cache the result for each scraping operation, and the return the cache on the http-endpoint.

var cache={};

app.get('/myendpoint', function(req, res) {
    res.json(cache);
})

request(url, function(error, response, html){
    ...
    setTimeout(function () {
        ...
            console.log(result);
            //set cache here
            cache=result;
            browser.exit();
        });
   }, 5000);
});

app.listen(1338);

If you want to cache to update, wrap the scraping function in a setInterval.

var cache={};

app.get('/myendpoint', function(req, res) {
    res.json(cache);
})

function updateCache() {
    request(url, function(error, response, html){
        ...
        setTimeout(function () {
            ...
                console.log(result);
                //set cache here
                cache=result;
                browser.exit();
            });
       }, 5000);
    });
}

//Update cache every 60 secs.
setInterval(updateCache, 60000);

app.listen(1338);