How to scrape .aspx site

Question

I'm trying to scrape: http://search.bgc-group.com/Default.aspx

When I'm running my code, I'm always getting the first page.

My code:

var casper = require('casper').create({
        verbose: true,
        logLevel: "debug",
        type: 'xpath',
    });
casper.start('http://search.bgc-group.com/Default.aspx');

casper.thenOpen('http://search.bgc-group.com/Default.aspx', {
    method: 'post',
    data:   {
        '__EVENTARGUMENT': 'Page$4',
        '__EVENTTARGET':  'ctl00$body$ctlJobListing1$gvJobListing',
        '__EVENTVALIDATION':  '',
        '__PREVIOUSPAGE':  '',
        '__VIEWSTATE':  ''
    }
});

casper.then(function() {
    this.echo('POSTED it.');
    this.capture("test.png")
});

casper.run();

I've not left the __EVENTVALIDATION, __PREVIOUSPAGE and __VIEWSTATE fields blank in the actual code. Just skipped here as they were very long string.

Answer 1

You can do the following instead of the casper.thenOpen block:

var x = require('casper').selectXPath;
casper.thenClick("a[href*='Page$4']");
casper.waitForSelector(x("//td/span[text()='4']")); // might be unnecessary