May be this helps
SELECT SQL_CALC_FOUND_ROWS
u.id_user AS id,
i.id_user_from,
i.id_user_to,
u.name AS name,
UNIX_TIMESTAMP(i.date_msg) AS date_msg,
i.message AS msg
FROM inbox AS i
INNER JOIN user AS u ON u.id_user = IF(i.id_user_from = 1 /*me*/, i.id_user_to, i.id_user_from)
WHERE id_msg IN
(SELECT MAX(id_msg) AS id FROM
(
SELECT id_msg, id_user_from AS id_with
FROM inbox
WHERE id_user_to = 1
UNION ALL
SELECT id_msg, id_user_to AS id_with
FROM inbox
WHERE id_user_from = 1) AS t
GROUP BY id_with
)
ORDER BY i.id_msg DESC