You need to unproject screen point into an OpenGL world space:
vec3 UnProjectPoint( const vec3& Point, const max4& Projection, const mat4& ModelView )
{
vec4 R( Point, 1.0f );
R.x = 2.0f * R.x - 1.0f;
R.y = 2.0f * R.y - 1.0f;
R.y = -R.y;
R.z = 1.0f;
R = Projection.GetInversed() * R;
R = ModelView.GetInversed() * R;
return R.ToVec3();
}