Your key word is "arc ball" control. the pseudocode below illustrate it
/**
help function for arcball
orthogonal projection on sphere of radius 1,
standing in (x = 0, y = 0)
*/
inline vector3 ortho_project_on_sphere( float x , float y )
{
vector3 p(x, y, 0);
float ls = p.len_squared();
if ( ls >= 1.0f )
p.norm();
else
p.z = (float)sqrt(1.0f - ls);
return p;
}
//-------------------------------------------------------------------------------
/*
calculate rotation of arcball user input,
used to perform object rotation by mouse.
"from" and "to" the mouse on screen coordinates (with - x,y on screen)
in range of -1 to +1
the arcball radius is 1.0 and it stand in a center (x = 0, y = 0)
*/
inline xxquaternion arcball(const vector3& from, const vector3& to)
{
vector3 p_f = ortho_project_on_sphere(from.x, from.y );
vector3 p_t = ortho_project_on_sphere(to.x, to.y );
vector3 c = cross(p_f, p_t);
float d = dot(p_f, p_t);
return xxquaternion(c.x, c.y, c.z, d);
}