Best way is to remove modal div when you are creating new modal div.
$('#contact').on('click', function () {
$('#modal').remove(); //remove modal here
$('body').append('<div id="modal"></div>');
$("#modal").load('/forms/contact-form.html').hide().fadeIn(1000);
});