Just get the index of the last visible LI, and another 5 to show
jQuery(function($) {
$('.layered-items').each(function(){
var x = 5;
$('li:lt('+x+')', this).show();
});
$('.loadmore-filter').click(function(){
var ul = $(this).closest('span').prev('ul'),
x = $('li:visible:last', ul).index(),
y = x + 6;
$('li:lt('+y+'):gt('+x+')', ul).show();
});
});