If you just want coloured points, something like this will do it:
palette(c("red","green","yellow","blue","black"))
plot.default(
as.data.frame.table(t(mat))[1:2],
col=findInterval(t(mat),c(0,0.2,0.4,0.6,0.9)),
pch=19,
axes=FALSE,ann=FALSE,
panel.first=grid()
)
axis(2,at=1:length(rownames(mat)),labels=rownames(mat),lwd=0,lwd.ticks=1,las=1)
axis(1,at=1:length(colnames(mat)),labels=colnames(mat),lwd=0,lwd.ticks=1)
box()
palette("default")
Result: