I am trying to scrape book genre information from google.
Much like when you put a calculation/conversion into google, the result displays in a box above the search results. I can scrape the data within this box in the browser (console) quite easily, however when I attempt the same code within casper, the content boxes don't appear anywhere in the code. The only way I can replicate this in browser is if I turn JS off.
I'm not sure why a different format is displayed to CasperJS and my own browser, but is there a way to get them to be the same? Here is the current code I am using, where
casper.start();
casper.userAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_4) AppleWebKit/537.36 (KHTML, like Gecko) ');
casper.thenOpen(links, function() {
casper.waitForSelector('.answer_predicate', function() {
this.echo(this.getHTML('.answer_predicate'));
});
});
casper.run();
}
Running the following:
var casper = require('casper').create({
pageSettings: {
loadImages: false,
loadPlugins: false,
userAgent: 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1588.0 Safari/537.36'
}
});
links = 'https://www.google.com/webhp?hl=en&tab=ww#safe=off&hl=en&output=search&sclient=psy-ab&q=The+Love+Affairs+of+a+Bibliomaniac+book+genre&oq=The+Love+Affairs+of+a+Bibliomaniac+book+genre'
casper.start();
casper.thenOpen(links, function() {
this.waitForSelector('.answer_predicate', function() {
this.echo(this.getHTML('.answer_predicate'));
this.echo(this.getElementInfo('.answer_predicate').text);
});
});
casper.run();
Gives me this output:
<span class="kno-a-v">Fiction</span>
Fiction
My assumption is that the problem here is the same as the one posted here.