The problem was that your animations were running over each other. You can prevent it with something like this:
var $nav = $('#nav-logo');
var $window = $(window);
var wasTop = true;
$window.on('scroll', function () {
var isTop = !$window.scrollTop();
if (isTop && !wasTop) {
$nav.stop();
$nav.transition({
scale: 1.0,
y: 0
});
wasTop = true;
} else if (!isTop && wasTop) {
$nav.stop();
$nav.transition({
scale: 0.5,
y: -126
});
wasTop = false;
}
});