With the cursor placed inside the table, you can modify your code to look like this:
setup: function(editor) {
editor.addButton('mybutton', {
text: 'My button',
icon: false,
onclick: function() {
editor.execCommand('mceTableInsertRowAfter', false, editor);
}
});
}
See example - tinyMCE fiddle