I think the easiest way to calculate the diff in your case would be to calculate all the elements first and then remove the common ones (already calculated):
HashSet<Foo> all = listOfLists
.Skip(1)
.Aggregate(
new HashSet<Foo>(listOfLists.First()),
(h, e) =>
{
h.UnionWith(e);
return h;
}
);
HashSet<Foo> diff = all.ExceptWith(same)