I solved it like this:
// y is the drag amount
// visible height
var height = $(this).height()
// difference
var diff = (Math.abs(y) % height) / height
// I checked Hammer.js's source. It defines velocity as the change in 16ms.
// So multiplying it by 62.5 gives you the change in 1s.
// Not sure why I did that, but works pretty well in most situations
var inertia = event.gesture.velocityY * 62.5
if (
(event.gesture.direction == 'up' && diff + (inertia / height) >= 0.5) ||
(event.gesture.direction == 'down' && diff - (inertia / height) >= 0.5)
) {
// show the one below
} else {
// show the one above
}