How to retrieve the ajax data whose loading requires a mouse click with PhantomJS or other tools

Question

I'm using PhantomJS to retrieve this page: Target Page Link. The contents I need are under the "行政公告" and "就業徵才公告" tabs. Because this page is written in Chinese, in case you cannot find the tabs, you can use "find" function of the browsers to find the "行政公告" and "就業徵才公告" tabs. Because the contents under the "行政公告" tab are the loaded as the default option, I can easily use the script below to retrieve the page:

var page = require('webpage').create();
var url = 'http://sa.ttu.edu.tw/bin/home.php';
page.open(url, function (status) {
    var js = page.evaluate(function () {
    return document;
    });
    console.log(js.all[0].outerHTML); 
    phantom.exit();
});

But the contents under the "就業徵才公告" tab are not loaded after I use the PhamtomJS to emulate the mouse click with the code below:

var page = require('webpage').create();
var url = 'http://sa.ttu.edu.tw/bin/home.php';

page.open(url, function (status) {
page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js",                   function() {
    // jQuery is loaded, now manipulate the DOM
    $('#sm_adf63b93c89375a0bade42e5360b73274_1_Dyn_2_1').trigger('mouseover');
});
var js = page.evaluate(function () {
    return document;
});
    console.log(js.all[0].outerHTML); 
    phantom.exit();
});

This doesn't work as the contents under the "就業徵才公告" tab are not loaded. How should I do to retrieve the contents under the "就業徵才公告" tab?

Update:

After read a PhantomJS example, I refactored the code to below. It didn't work because the contents under the "就業徵才公告" tab are not loaded.

var page = require('webpage').create();
var address = 'http://sa.ttu.edu.tw/bin/home.php';

page.open(address, function (status) {
    if (status !== 'success') {
        console.log('Unable to load the address!');
        phantom.exit();
    } else {
        window.setTimeout(function () {
            var results = page.evaluate(function() {
                $('#sm_adf63b93c89375a0bade42e5360b73274_1_Dyn_2_1').trigger('mouseover');
                return document.documentElement.innerHTML;
            });

            console.log(results);

            phantom.exit();
        }, 5000);
    }
});

If any way could solve this problem is welcomed. Not limited to PhamtoJS.

Answer 1

Tested this code, and it outputs the correct image with the desired tab selected. It wasn't so straightforward because of the underlying structure of the page. Hopefully you can use this as a bit of a learning exercise in processing the DOM.

// utility function to send mouseclick event to an element
function mouseclick( element ) {
    // create a mouse click event
    var event = document.createEvent( 'MouseEvents' );
    event.initMouseEvent( 'click', true, true, window, 1, 0, 0 );

    // send click to element
    element.dispatchEvent( event );
}

// final function called, output screenshot, exit
function after_clicked( page ) {
    console.log( "+after_clicked()" );

    page.render( "after_click.png" );
    console.log( "Done" );
    phantom.exit( 0 );
}

// middle function, click on desired tab
function click_div( page ) {
    console.log( "+click_div()" );

    var clicked = page.evaluate(
        function ( mouseclick_fn ) {
            // want the div with class "submenu"
            var div = document.querySelector( "div.submenu" );
            if ( ! div ) {
                return false;
            }

            // want all the list elements in the div
            var li_array = div.querySelectorAll( "li" );
            if ( ! li_array ) {
                return false;
            }

            // we want the 2nd list element
            var li2 = li_array[1];
            if ( ! li2 ) {
                return false;
            }

            // want the anchor inside the 2nd list element
            var anchor = li2.querySelector( "a" );
            if ( ! anchor ) {
                return false;
            }

            // must focus on anchor to trigger underlying javascript on page
            anchor.focus();

            // want the div within this anchor, so we can click on the div
            var element = anchor.querySelector( "div" );
            if ( ! element ) {
                return false;
            }

            // click on this inner div
            mouseclick_fn( element );
            return true;
        }, mouseclick
    );

    if ( ! clicked ) {
        console.log( "Failed to find desired element" );
        phantom.exit( 1 );
        return;
    }

    console.log( "- clicked, waiting 5 seconds" );
    window.setTimeout(
        function () {
            after_clicked( page );
        },
        5000
    );
}

// first function, create page, load page
function main() {
    console.log( "+main()" );

    var page = require('webpage').create();

    page.open(
        "http://sa.ttu.edu.tw/bin/home.php",
        function (status) {
            if ( status !== 'success' ) {
                console.log( "Failed" );
                phantom.exit( 1 );
                return;
            }

            console.log( "- page loaded, waiting 2 seconds..." );
            window.setTimeout(
                function () {
                    click_div( page );
                },
                2000
            );
        }
    );
}

main();