I scrapped the page https://www.wikifolio.com/de/de/home for a while now with casperjs. Recently it takes a login to see the information on the page and I just can't get it to work. I can't seem to find which item I have to click to get rid of the disclaimer and later to log into the site.
It's possible with 2 different ways, this will work:
var casper = require('casper').create({
verbose: true,
logLevel: 'debug',
waitTimeout: 5000,
userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0',
viewportSize:{width: 1600, height: 900}
});
casper
.on("error", function(msg){ this.echo("error: " + msg, "ERROR") })
.on("page.error", function(msg, trace){ this.echo("Page Error: " + msg, "ERROR") })
.on("remote.message", function(msg){ this.echo("Info: " + msg, "INFO") });
casper
.start("https://www.wikifolio.com/de/de/home", function(){
this.click('div.js-change-country-mode-btn');
this.wait(500,function(){this.click('a.js-login-button')});
this.wait(500,function(){
this.fillSelectors('form[action$=login]', {
"input#Username" : "[email protected]",
"input#Password" : "<pass>"
},true);
})
})
.then(function(){
this
.capture("Afterlogin.png")
.wait(5000,function(){ this.capture("Afterlogin2.png") })
})
.run();
You may use sendKeys()
Instead of fillSelectors().
File: Afterlogin.png
File: Afterlogin2.png
This will work too:
You can do it by using cookie:
var casper = require('casper').create({
verbose: true,
logLevel: 'debug',
waitTimeout: 5000,
userAgent: 'Mozilla/5.0 (X11; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0',
viewportSize:{width: 1600, height: 900}
});
casper
.on("error", function(msg){ this.echo("error: " + msg, "ERROR") })
.on("page.error", function(msg, trace){ this.echo("Page Error: " + msg, "ERROR") })
.on("remote.message", function(msg){ this.echo("Info: " + msg, "INFO") });
//for www.wikifolio.com/de/de/home auth
phantom.cookies = [{// an array of objects
'name' : 'theAuthCookie',
'value' : '<very long string>',
'domain' : 'www.wikifolio.com',
'path' : '/',
'httponly' : false,
'secure' : true,
'expires' : (new Date()).getTime() + (1000 * 60 * 60 * 43800) //5 years
},{ 'name' : 'DisclaimerCountryPopupV2',
'value' : 'de',
'domain' : 'www.wikifolio.com',
'path' : '/',
'httponly' : false,
'secure' : true,
'expires' : (new Date()).getTime() + (1000 * 60 * 60 * 43800) }]
var target = "https://www.wikifolio.com/de/de/alle-wikifolios/suche#/?tags=aktde,akteur,aktusa,akthot,aktint,etf,fonds,anlagezert,hebel&media=true&private=true&assetmanager=true&theme=true&super=true&WithoutLeverageProductsOnly=true&languageOnly=true"
casper
.start(target, function(){ })
.then(function(){
this
.capture("Afterlogin.png")
.wait(5000,function(){ this.capture("Afterlogin2.png") })
})
.run();