Introduce a new variable @cat. Compare with catid and reset the @pos on catid change
SET @pos := 0;
SET @cat:= 0;
UPDATE products p join
(
select id,
@pos:=if(@cat<>catid,0,@pos+1) as new_order,
@cat:=catid
from products
ORDER BY order
) p2 on p.id=p2.id
SET order = p2.new_order;