You can return a button instead of the span element and have a function bound onClick event. For example :
{
name: 'Notes',
width: 60,
sortable: false, resizable: false,
search: false,
formatter: function() {
return "<button onclick='OpenDialog()' style='margin-left:12px'>Pop Up Dlg</button>";
}
},
and then the function you need to call
function OpenDialog(){
//Assuming you have Jquery Dialog already set. (http://jqueryui.com/dialog/)
$("#myDialogBox").dialog("open");
}