Callback/Promises Implementation in Express.js

Question

I'm writing an Express.js application using x-ray as a scraper to get some info.

I want to create a model for each website I'm scraping (different website = different data/procedures to scrape). Here's the code of the module:

module.exports.getData = function(title){
var Xray = require('x-ray');  
var x = Xray();


var url = "http://www.mywebsite.it/online/search?text="+title;
var scraped = '';
x(url, '.listing-products.listing-rows.clearfix h2.title',
  [{
    title: 'a',
    link: 'a@href' 
  }]
)(function (err, obj) {
    for (var i = obj.length - 1; i >= 0; i--) {
        scraped += obj[i]['title'] + " "; //we get just the title of the links for now
    };
    sendScraped(scraped);
});

}

problem is that the controller's function getData does not return anything because it is called and the execution goes forward without waiting the completation of the x() scraping function.

I'm trying to implement a callback function sendScraped(scraped) to get my controller to wait for the completation, but i don't know how to call for it from the model. This is what i tried in the controller:

var mywebsite = require('../models/mywebsite')

exports.searchTitle = function(req, res) {
    mywebsite.getData(req.params.title);
};


global.sendScraped = function endScraping(data) {
    return res.send(data);
}

Answer 1

I'd suggest you change your getData() method to take a callback. It can then call that callback with the scraped data:

module.exports.getData = function (title, callback) {
    var Xray = require('x-ray');
    var x = Xray();
    var url = "http://www.mywebsite.it/online/search?text=" + title;
    var scraped = '';
    x(url, '.listing-products.listing-rows.clearfix h2.title', [{
        title: 'a',
        link: 'a@href'
    }])(function (err, obj) {
        if (err) return callback(err);
        for (var i = obj.length - 1; i >= 0; i--) {
            scraped += obj[i]['title'] + " "; //we get just the title of the links for now
        };
        callback(null, scraped);
    });
}

You can then use that in your request handler like this:

var mywebsite = require('../models/mywebsite')

exports.searchTitle = function(req, res) {
    mywebsite.getData(req.params.title, function(err, scrapeData) {
       if (err) {
           // do something with error
       } else {
           res.send(scrapeData);
       }
    });
};

---

And, here's a version using promises:

var Xray = require('x-ray');

function xp(url, selector, args) {
    return new Promise(function(resolve, reject) {
        var x = Xray();
        x(url, selector, args)(function(err, obj) {
            if (err) {
                reject(err);
            } else {
                resolve(obj);
            }
        });        
    });
}

module.exports.getData = function (title) {
    var url = "http://www.mywebsite.it/online/search?text=" + title;
    var selector = '.listing-products.listing-rows.clearfix h2.title';
    return xp(url, selector, [{title: 'a', link: 'a@href'}]).then(function(obj) {
        var scraped = "";
        for (var i = obj.length - 1; i >= 0; i--) {
            scraped += obj[i]['title'] + " "; //we get just the title of the links for now
        }
        return scraped;        
    });
}

You can then use that in your request handler like this:

var mywebsite = require('../models/mywebsite');

exports.searchTitle = function(req, res) {
    mywebsite.getData(req.params.title).then(function(scrapeData) {
           res.send(scrapeData);
    }, function(err) {
        // handle error here
    });
};