You are about to rotate about the center and get the new corner coordinate. Assuming you want to get the largest bounding box with the angle is not 90 degree:
var theta = layer.rotate*Math.PI/180.;
// Find the middle rotating point
var midX = layer.x + layer.width/2;
var midY = layer.y + layer.height/2;
// Find all the corners relative to the center
var cornersX = [layer.x-midX, layer.x-midX, layer.x+layer.width-midX, layer.x+layer.width-midX];
var cornersY = [layer.y-midY, layer.y+layer.height-midY, midY-layer.y, layer.y+layer.height-midY];
// Find new the minimum corner X and Y by taking the minimum of the bounding box
var newX = 1e10;
var newY = 1e10;
for (var i=0; i<4; i=i+1) {
newX = min(newX, cornersX[i]*Math.cos(theta) - cornersY[i]*Math.sin(theta) + midX);
newY = min(newY, cornersX[i]*Math.sin(theta) + cornersY[i]*Math.cos(theta) + midY);
}
// new width and height
newWidth = midX - newX;
newHeight = midY - newY;