So I solved it by adding a callback function to the slideToggle that checks if the div is visible.
Like this:
$("#download").click(function() {
$("#musicDownload").slideToggle(500, function(){
if($("#musicDownload").is(":visible")){
$("html, body").animate({scrollTop: $("#download").offset().top},500);
}
});
return false;
});