If you want the one with the max priority, then you need to change your subquery pretty significantly to choose the MerchID with the highest priority:
SELECT ID FROM tbl_a WHERE fld_merchID IN (
SELECT DISTINCT MerchID
FROM tbl_b
INNER JOIN (
SELECT Name, MAX(Priority) as Priority
FROM tbl_b
WHERE Name LIKE 'AAA.com'
GROUP BY Name
) mx ON mx.Name = tbl_b.Name AND mx.Priority = tbl_b.Priority
)