It appears your values are declared inside the function and are out of scope.
Do this:
var selectedtile = new Object() //de OUTSIDE All functions.
//de this works too: var selectedtile = {}
function testTile(mouseX, mouseY){
if(mouseX<mapArray[0].length*20 && mouseY<mapArray.length*20){
var tile = getTilePosition(mouseX-10, mouseY-10);
selectedtile.x = tile.xtile,
selectedtile.y = tile.ytile;
}
}
function getTilePosition(x, y){
var xtile = Math.floor(x/20);
var ytile = Math.floor(y/20);
return {
xtile : xtile,
ytile : ytile
}
}